diff --git a/example/src/tstream.c b/example/src/tstream.c index 62d94b041d5b304e2d4f449436fcf9896a97e395..8ffa932bd29d6f89441c7f045611f42e058dd872 100644 --- a/example/src/tstream.c +++ b/example/src/tstream.c @@ -25,7 +25,7 @@ int32_t init_env() { return -1; } - TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1"); + TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2"); if (taos_errno(pRes) != 0) { printf("error in create db, reason:%s\n", taos_errstr(pRes)); return -1; @@ -63,7 +63,7 @@ int32_t init_env() { } int32_t create_stream() { - printf("create topic\n"); + printf("create stream\n"); TAOS_RES* pRes; TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); if (pConn == NULL) { @@ -77,7 +77,9 @@ int32_t create_stream() { } taos_free_result(pRes); - const char* sql = "select ts,k from tu1"; + /*const char* sql = "select min(k), max(k), sum(k) from tu1";*/ + const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1"; + /*const char* sql = "select sum(k) from tu1 interval(10m)";*/ pRes = tmq_create_stream(pConn, "stream1", "out1", sql); if (taos_errno(pRes) != 0) { printf("failed to create stream out1, reason:%s\n", taos_errstr(pRes)); diff --git a/include/common/taosdef.h b/include/common/taosdef.h index f252143fa639629dd7deeea551078983addb7bd3..797ebf29c59c38bc4fd5af1143976037360843b1 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -63,9 +63,12 @@ typedef enum { } ETsdbStatisStatus; typedef enum { - TSDB_SMA_STAT_OK = 0, // ready to provide service - TSDB_SMA_STAT_EXPIRED = 1, // not ready or expired -} ETsdbSmaStat; + TSDB_SMA_STAT_UNKNOWN = -1, // unknown + TSDB_SMA_STAT_OK = 0, // ready to provide service + TSDB_SMA_STAT_EXPIRED = 1, // not ready or expired + TSDB_SMA_STAT_DROPPED = 2, // sma dropped +} ETsdbSmaStat; // bit operation + typedef enum { TSDB_SMA_TYPE_BLOCK = 0, // Block-wise SMA diff --git a/include/common/tcommon.h b/include/common/tcommon.h index c91efc3ce2f2224d3690f9f559f24173a9452660..eb9f45087295841fbf03e580955d430f510522f0 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -54,13 +54,16 @@ typedef struct SColumnDataAgg { } SColumnDataAgg; typedef struct SDataBlockInfo { - STimeWindow window; - int32_t rows; - int32_t rowSize; - int16_t numOfCols; - int16_t hasVarCol; - union {int64_t uid; int64_t blockId;}; - int64_t groupId; // no need to serialize + STimeWindow window; + int32_t rows; + int32_t rowSize; + int16_t numOfCols; + int16_t hasVarCol; + union { + int64_t uid; + int64_t blockId; + }; + int64_t groupId; // no need to serialize } SDataBlockInfo; typedef struct SSDataBlock { @@ -92,7 +95,7 @@ int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock); void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock); int32_t tEncodeDataBlocks(void** buf, const SArray* blocks); -void* tDecodeDataBlocks(const void* buf, SArray* blocks); +void* tDecodeDataBlocks(const void* buf, SArray** blocks); static FORCE_INLINE void blockDestroyInner(SSDataBlock* pBlock) { // WARNING: do not use info.numOfCols, @@ -101,16 +104,16 @@ static FORCE_INLINE void blockDestroyInner(SSDataBlock* pBlock) { for (int32_t i = 0; i < numOfOutput; ++i) { SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - tfree(pColInfoData->varmeta.offset); + taosMemoryFreeClear(pColInfoData->varmeta.offset); } else { - tfree(pColInfoData->nullbitmap); + taosMemoryFreeClear(pColInfoData->nullbitmap); } - tfree(pColInfoData->pData); + taosMemoryFreeClear(pColInfoData->pData); } taosArrayDestroy(pBlock->pDataBlock); - tfree(pBlock->pBlockAgg); + taosMemoryFreeClear(pBlock->pBlockAgg); } static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { blockDestroyInner(pBlock); } @@ -144,7 +147,7 @@ static FORCE_INLINE void* tDecodeSMqPollRsp(void* buf, SMqPollRsp* pRsp) { buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); buf = taosDecodeFixedI32(buf, &pRsp->numOfTopics); if (pRsp->numOfTopics == 0) return buf; - pRsp->schema = (SSchemaWrapper*)calloc(1, sizeof(SSchemaWrapper)); + pRsp->schema = (SSchemaWrapper*)taosMemoryCalloc(1, sizeof(SSchemaWrapper)); if (pRsp->schema == NULL) return NULL; buf = tDecodeSSchemaWrapper(buf, pRsp->schema); buf = taosDecodeFixedI32(buf, &sz); @@ -160,9 +163,9 @@ static FORCE_INLINE void* tDecodeSMqPollRsp(void* buf, SMqPollRsp* pRsp) { static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqPollRsp* pRsp) { if (pRsp->schema) { if (pRsp->schema->nCols) { - tfree(pRsp->schema->pSchema); + taosMemoryFreeClear(pRsp->schema->pSchema); } - free(pRsp->schema); + taosMemoryFree(pRsp->schema); } taosArrayDestroyEx(pRsp->pBlockData, (void (*)(void*))blockDestroyInner); pRsp->pBlockData = NULL; diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index f181c26e92aed6a14576a69ef97d7e885f5d4b6b..e6ddc1e5b50b5668a6c651011054111333ee75e2 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -102,7 +102,8 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u : ((p1_)->pData + ((r_) * (p1_)->info.bytes))) int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); -int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, uint32_t numOfRow2); +int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, + uint32_t numOfRow2); int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows); int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock); @@ -113,7 +114,8 @@ size_t blockDataGetNumOfCols(const SSDataBlock* pBlock); size_t blockDataGetNumOfRows(const SSDataBlock* pBlock); int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc); -int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, int32_t pageSize); +int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, + int32_t pageSize); int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock); int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf); @@ -136,6 +138,8 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock); size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); void* blockDataDestroy(SSDataBlock* pBlock); +void blockDebugShowData(SArray* dataBlocks); + #ifdef __cplusplus } #endif diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 89a0e6b4dc5adf1659fc2a582b64cc355b2c9c89..698352f63652f9e57ae9d44493358a2b3f1cbc85 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -93,7 +93,7 @@ typedef struct { #define schemaFLen(s) ((s)->flen) #define schemaVLen(s) ((s)->vlen) #define schemaColAt(s, i) ((s)->columns + i) -#define tdFreeSchema(s) tfree((s)) +#define tdFreeSchema(s) taosMemoryFreeClear((s)) STSchema *tdDupSchema(const STSchema *pSchema); int32_t tdEncodeSchema(void **buf, STSchema *pSchema); @@ -493,7 +493,7 @@ typedef struct { #define kvRowCpy(dst, r) memcpy((dst), (r), kvRowLen(r)) #define kvRowColVal(r, colIdx) POINTER_SHIFT(kvRowValues(r), (colIdx)->offset) #define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i)) -#define kvRowFree(r) tfree(r) +#define kvRowFree(r) taosMemoryFreeClear(r) #define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r)) #define kvRowValLen(r) (kvRowLen(r) - TD_KV_ROW_HEAD_SIZE - sizeof(SColIdx) * kvRowNCols(r)) #define kvRowTKey(r) (*(TKEY *)(kvRowValues(r))) @@ -593,7 +593,7 @@ SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder); static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, int8_t type, const void *value) { if (pBuilder->nCols >= pBuilder->tCols) { pBuilder->tCols *= 2; - SColIdx *pColIdx = (SColIdx *)realloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols); + SColIdx *pColIdx = (SColIdx *)taosMemoryRealloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols); if (pColIdx == NULL) return -1; pBuilder->pColIdx = pColIdx; } @@ -608,7 +608,7 @@ static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t col while (tlen > pBuilder->alloc - pBuilder->size) { pBuilder->alloc *= 2; } - void *buf = realloc(pBuilder->buf, pBuilder->alloc); + void *buf = taosMemoryRealloc(pBuilder->buf, pBuilder->alloc); if (buf == NULL) return -1; pBuilder->buf = buf; } diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 583496a4c673b5364f462d65d644d7a5524c9b1d..136a576c7cbdbbdbf7b2a975dfbb2aca27060b84 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -23,8 +23,8 @@ #include "tencode.h" #include "thash.h" #include "tlist.h" -#include "trow.h" #include "tname.h" +#include "trow.h" #include "tuuid.h" #ifdef __cplusplus @@ -109,6 +109,7 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_STREAMTABLES, TSDB_MGMT_TABLE_TP, TSDB_MGMT_TABLE_FUNC, + TSDB_MGMT_TABLE_INDEX, TSDB_MGMT_TABLE_MAX, } EShowType; @@ -183,6 +184,13 @@ typedef struct SField { int32_t bytes; } SField; +typedef struct SRetention { + int32_t freq; + int32_t keep; + int8_t freqUnit; + int8_t keepUnit; +} SRetention; + #pragma pack(push, 1) // null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta @@ -193,21 +201,13 @@ typedef struct SEp { typedef struct { int32_t contLen; - union { - int32_t vgId; - int32_t streamTaskId; - }; + int32_t vgId; } SMsgHead; -typedef struct { - int32_t workerType; - int32_t streamTaskId; -} SStreamExecMsgHead; - // Submit message for one table typedef struct SSubmitBlk { int64_t uid; // table unique id - int32_t tid; // table id + int64_t suid; // stable id int32_t padding; // TODO just for padding here int32_t sversion; // data schema version int32_t dataLen; // data part length, not including the SSubmitBlk head @@ -232,12 +232,12 @@ typedef struct { } SSubmitBlkIter; typedef struct { - int32_t totalLen; - int32_t len; - void* pMsg; + int32_t totalLen; + int32_t len; + const void* pMsg; } SSubmitMsgIter; -int32_t tInitSubmitMsgIter(SSubmitReq* pMsg, SSubmitMsgIter* pIter); +int32_t tInitSubmitMsgIter(const SSubmitReq* pMsg, SSubmitMsgIter* pIter); int32_t tGetSubmitMsgNext(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock); int32_t tInitSubmitBlkIter(SSubmitBlk* pBlock, SSubmitBlkIter* pIter); STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter); @@ -268,11 +268,18 @@ typedef struct SSchema { typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t igExists; + float xFilesFactor; + int32_t aggregationMethod; + int32_t delay; + int32_t ttl; int32_t numOfColumns; int32_t numOfTags; - SArray* pColumns; - SArray* pTags; - char comment[TSDB_STB_COMMENT_LEN]; + int32_t numOfSmas; + int32_t commentLen; + SArray* pColumns; // array of SField + SArray* pTags; // array of SField + SArray* pSmas; // array of SField + char* comment; } SMCreateStbReq; int32_t tSerializeSMCreateStbReq(void* buf, int32_t bufLen, SMCreateStbReq* pReq); @@ -462,7 +469,9 @@ typedef struct { int32_t tz; // query client timezone char intervalUnit; char slidingUnit; - char offsetUnit; + char + offsetUnit; // TODO Remove it, the offset is the number of precision tickle, and it must be a immutable duration. + int8_t precision; int64_t interval; int64_t sliding; int64_t offset; @@ -472,10 +481,9 @@ typedef struct { int32_t code; } SQueryTableRsp; -int32_t tSerializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pRsp); - -int32_t tDeserializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pRsp); +int32_t tSerializeSQueryTableRsp(void* buf, int32_t bufLen, SQueryTableRsp* pRsp); +int32_t tDeserializeSQueryTableRsp(void* buf, int32_t bufLen, SQueryTableRsp* pRsp); typedef struct { char db[TSDB_DB_FNAME_LEN]; @@ -499,10 +507,13 @@ typedef struct { int8_t cacheLastRow; int8_t ignoreExist; int8_t streamMode; + int32_t numOfRetensions; + SArray* pRetensions; // SRetention } SCreateDbReq; int32_t tSerializeSCreateDbReq(void* buf, int32_t bufLen, SCreateDbReq* pReq); int32_t tDeserializeSCreateDbReq(void* buf, int32_t bufLen, SCreateDbReq* pReq); +void tFreeSCreateDbReq(SCreateDbReq* pReq); typedef struct { char db[TSDB_DB_FNAME_LEN]; @@ -747,11 +758,13 @@ typedef struct { int8_t selfIndex; int8_t streamMode; SReplica replicas[TSDB_MAX_REPLICA]; - + int32_t numOfRetensions; + SArray* pRetensions; // SRetention } SCreateVnodeReq, SAlterVnodeReq; int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq); int32_t tDeserializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq); +int32_t tFreeSCreateVnodeReq(SCreateVnodeReq* pReq); typedef struct { int32_t vgId; @@ -888,14 +901,14 @@ typedef struct { } SRetrieveTableRsp; typedef struct { - int64_t handle; - int64_t useconds; - int8_t completed; // all results are returned to client - int8_t precision; - int8_t compressed; - int32_t compLen; - int32_t numOfRows; - char data[]; + int64_t handle; + int64_t useconds; + int8_t completed; // all results are returned to client + int8_t precision; + int8_t compressed; + int32_t compLen; + int32_t numOfRows; + char data[]; } SRetrieveMetaTableRsp; typedef struct { @@ -1147,7 +1160,7 @@ typedef struct { typedef struct { char name[TSDB_TOPIC_FNAME_LEN]; - char outputTbName[TSDB_TABLE_NAME_LEN]; + char outputSTbName[TSDB_TABLE_FNAME_LEN]; int8_t igExists; char* sql; char* ast; @@ -1296,7 +1309,7 @@ typedef struct { } SMqRebSubscribe; static FORCE_INLINE SMqRebSubscribe* tNewSMqRebSubscribe(const char* key) { - SMqRebSubscribe* pRebSub = (SMqRebSubscribe*)calloc(1, sizeof(SMqRebSubscribe)); + SMqRebSubscribe* pRebSub = (SMqRebSubscribe*)taosMemoryCalloc(1, sizeof(SMqRebSubscribe)); if (pRebSub == NULL) { goto _err; } @@ -1318,7 +1331,7 @@ _err: taosArrayDestroy(pRebSub->lostConsumers); taosArrayDestroy(pRebSub->removedConsumers); taosArrayDestroy(pRebSub->newConsumers); - tfree(pRebSub); + taosMemoryFreeClear(pRebSub); return NULL; } @@ -1425,12 +1438,11 @@ int32_t tSerializeSVCreateTbBatchReq(void** buf, SVCreateTbBatchReq* pReq); void* tDeserializeSVCreateTbBatchReq(void* buf, SVCreateTbBatchReq* pReq); typedef struct { - SArray* rspList; // SArray + SArray* rspList; // SArray } SVCreateTbBatchRsp; -int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp); -int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp); - +int32_t tSerializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp); +int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp); typedef struct { int64_t ver; @@ -1620,7 +1632,7 @@ static FORCE_INLINE void tFreeReqKvHash(SHashObj* info) { void* pIter = taosHashIterate(info, NULL); while (pIter != NULL) { SKv* kv = (SKv*)pIter; - tfree(kv->value); + taosMemoryFreeClear(kv->value); pIter = taosHashIterate(info, pIter); } } @@ -1643,13 +1655,13 @@ static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq, bool deep) { } else { taosArrayDestroy(req->reqs); } - free(pReq); + taosMemoryFree(pReq); } static FORCE_INLINE void tFreeClientKv(void* pKv) { SKv* kv = (SKv*)pKv; if (kv) { - tfree(kv->value); + taosMemoryFreeClear(kv->value); } } @@ -1676,7 +1688,7 @@ static FORCE_INLINE int32_t tEncodeSKv(SCoder* pEncoder, const SKv* pKv) { static FORCE_INLINE int32_t tDecodeSKv(SCoder* pDecoder, SKv* pKv) { if (tDecodeI32(pDecoder, &pKv->key) < 0) return -1; if (tDecodeI32(pDecoder, &pKv->valueLen) < 0) return -1; - pKv->value = malloc(pKv->valueLen + 1); + pKv->value = taosMemoryMalloc(pKv->valueLen + 1); if (pKv->value == NULL) return -1; if (tDecodeCStrTo(pDecoder, (char*)pKv->value) < 0) return -1; return 0; @@ -1930,7 +1942,7 @@ static FORCE_INLINE int32_t tEncodeSSchemaWrapper(void** buf, const SSchemaWrapp static FORCE_INLINE void* tDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW) { buf = taosDecodeFixedU32(buf, &pSW->nCols); - pSW->pSchema = (SSchema*)calloc(pSW->nCols, sizeof(SSchema)); + pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); if (pSW->pSchema == NULL) { return NULL; } @@ -1940,12 +1952,47 @@ static FORCE_INLINE void* tDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW) } return buf; } + +typedef struct { + char name[TSDB_TABLE_FNAME_LEN]; + char stb[TSDB_TABLE_FNAME_LEN]; + int8_t igExists; + int8_t intervalUnit; + int8_t slidingUnit; + int8_t timezone; + int32_t dstVgId; // for stream + int64_t interval; + int64_t offset; + int64_t sliding; + int32_t exprLen; // strlen + 1 + int32_t tagsFilterLen; // strlen + 1 + int32_t sqlLen; // strlen + 1 + int32_t astLen; // strlen + 1 + char* expr; + char* tagsFilter; + char* sql; + char* ast; +} SMCreateSmaReq; + +int32_t tSerializeSMCreateSmaReq(void* buf, int32_t bufLen, SMCreateSmaReq* pReq); +int32_t tDeserializeSMCreateSmaReq(void* buf, int32_t bufLen, SMCreateSmaReq* pReq); +void tFreeSMCreateSmaReq(SMCreateSmaReq* pReq); + +typedef struct { + char name[TSDB_TABLE_FNAME_LEN]; + int8_t igNotExists; +} SMDropSmaReq; + +int32_t tSerializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq); +int32_t tDeserializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq); + typedef struct { int8_t version; // for compatibility(default 0) int8_t intervalUnit; // MACRO: TIME_UNIT_XXX int8_t slidingUnit; // MACRO: TIME_UNIT_XXX + int8_t timezoneInt; // sma data expired if timezone changes. char indexName[TSDB_INDEX_NAME_LEN]; - char timezone[TD_TIMEZONE_LEN]; // sma data expired if timezone changes. + char timezone[TD_TIMEZONE_LEN]; int32_t exprLen; int32_t tagsFilterLen; int64_t indexUid; @@ -2030,8 +2077,8 @@ typedef struct { static FORCE_INLINE void tdDestroyTSma(STSma* pSma) { if (pSma) { - tfree(pSma->expr); - tfree(pSma->tagsFilter); + taosMemoryFreeClear(pSma->expr); + taosMemoryFreeClear(pSma->tagsFilter); } } @@ -2041,19 +2088,24 @@ static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW) { for (uint32_t i = 0; i < pSW->number; ++i) { tdDestroyTSma(pSW->tSma + i); } - tfree(pSW->tSma); + taosMemoryFreeClear(pSW->tSma); } } } +static FORCE_INLINE void tdFreeTSmaWrapper(STSmaWrapper* pSW) { + tdDestroyTSmaWrapper(pSW); + taosMemoryFreeClear(pSW); +} + static FORCE_INLINE int32_t tEncodeTSma(void** buf, const STSma* pSma) { int32_t tlen = 0; tlen += taosEncodeFixedI8(buf, pSma->version); tlen += taosEncodeFixedI8(buf, pSma->intervalUnit); tlen += taosEncodeFixedI8(buf, pSma->slidingUnit); + tlen += taosEncodeFixedI8(buf, pSma->timezoneInt); tlen += taosEncodeString(buf, pSma->indexName); - tlen += taosEncodeString(buf, pSma->timezone); tlen += taosEncodeFixedI32(buf, pSma->exprLen); tlen += taosEncodeFixedI32(buf, pSma->tagsFilterLen); tlen += taosEncodeFixedI64(buf, pSma->indexUid); @@ -2087,8 +2139,8 @@ static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) { buf = taosDecodeFixedI8(buf, &pSma->version); buf = taosDecodeFixedI8(buf, &pSma->intervalUnit); buf = taosDecodeFixedI8(buf, &pSma->slidingUnit); + buf = taosDecodeFixedI8(buf, &pSma->timezoneInt); buf = taosDecodeStringTo(buf, pSma->indexName); - buf = taosDecodeStringTo(buf, pSma->timezone); buf = taosDecodeFixedI32(buf, &pSma->exprLen); buf = taosDecodeFixedI32(buf, &pSma->tagsFilterLen); buf = taosDecodeFixedI64(buf, &pSma->indexUid); @@ -2121,7 +2173,7 @@ static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) { static FORCE_INLINE void* tDecodeTSmaWrapper(void* buf, STSmaWrapper* pSW) { buf = taosDecodeFixedU32(buf, &pSW->number); - pSW->tSma = (STSma*)calloc(pSW->number, sizeof(STSma)); + pSW->tSma = (STSma*)taosMemoryCalloc(pSW->number, sizeof(STSma)); if (pSW->tSma == NULL) { return NULL; } @@ -2131,7 +2183,7 @@ static FORCE_INLINE void* tDecodeTSmaWrapper(void* buf, STSmaWrapper* pSW) { for (uint32_t j = i; j >= 0; --i) { tdDestroyTSma(pSW->tSma + j); } - free(pSW->tSma); + taosMemoryFree(pSW->tSma); return NULL; } } @@ -2306,68 +2358,6 @@ static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* p } return buf; } - -enum { - STREAM_TASK_STATUS__RUNNING = 1, - STREAM_TASK_STATUS__STOP, -}; - -typedef struct { - void* inputHandle; - void* executor; -} SStreamRunner; - -typedef struct { - int64_t streamId; - int32_t taskId; - int32_t level; - int8_t status; - int8_t pipeSource; - int8_t pipeSink; - int8_t numOfRunners; - int8_t parallelizable; - SEpSet NextOpEp; - char* qmsg; - // not applied to encoder and decoder - SStreamRunner runner[8]; - // void* executor; - // void* stateStore; - // storage handle -} SStreamTask; - -static FORCE_INLINE SStreamTask* streamTaskNew(int64_t streamId, int32_t level) { - SStreamTask* pTask = (SStreamTask*)calloc(1, sizeof(SStreamTask)); - if (pTask == NULL) { - return NULL; - } - pTask->taskId = tGenIdPI32(); - pTask->status = STREAM_TASK_STATUS__RUNNING; - pTask->qmsg = NULL; - return pTask; -} - -int32_t tEncodeSStreamTask(SCoder* pEncoder, const SStreamTask* pTask); -int32_t tDecodeSStreamTask(SCoder* pDecoder, SStreamTask* pTask); -void tFreeSStreamTask(SStreamTask* pTask); - -typedef struct { - SMsgHead head; - SStreamTask* task; -} SStreamTaskDeployReq; - -typedef struct { - int32_t reserved; -} SStreamTaskDeployRsp; - -typedef struct { - SStreamExecMsgHead head; - SArray* data; // SArray -} SStreamTaskExecReq; - -typedef struct { - int32_t reserved; -} SStreamTaskExecRsp; - #pragma pack(pop) #ifdef __cplusplus diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index 54a145ff334f13f39ee657c24763cdbf5de9b5e1..64e399770b9343483764921f282247c67380b415 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -25,7 +25,15 @@ extern "C" { typedef struct SRpcMsg SRpcMsg; typedef struct SEpSet SEpSet; typedef struct SMgmtWrapper SMgmtWrapper; -typedef enum { QUERY_QUEUE, FETCH_QUEUE, WRITE_QUEUE, APPLY_QUEUE, SYNC_QUEUE, QUEUE_MAX } EQueueType; +typedef enum { + QUERY_QUEUE, + FETCH_QUEUE, + WRITE_QUEUE, + APPLY_QUEUE, + SYNC_QUEUE, + MERGE_QUEUE, + QUEUE_MAX, +} EQueueType; typedef int32_t (*PutToQueueFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq); typedef int32_t (*GetQueueSizeFp)(SMgmtWrapper* pWrapper, int32_t vgId, EQueueType qtype); diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 73a78131dc1cb6d81e85c3278328866d05ecd62a..8d49479d31e9989cc427cad3802f0571993d0e72 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -127,6 +127,8 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STB, "mnode-create-stb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STB, "mnode-alter-stb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_STB, "mnode-drop-stb", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_SMA, "mnode-create-sma", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_SMA, "mnode-drop-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TABLE_META, "mnode-table-meta", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_VGROUP_LIST, "mnode-vgroup-list", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_QNODE_LIST, "mnode-qnode-list", NULL, NULL) @@ -191,6 +193,10 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqCVConsumeReq, SMqCVConsumeRsp) TD_DEF_MSG_TYPE(TDMT_VND_TASK_DEPLOY, "vnode-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp) TD_DEF_MSG_TYPE(TDMT_VND_TASK_EXEC, "vnode-task-exec", SStreamTaskExecReq, SStreamTaskExecRsp) + TD_DEF_MSG_TYPE(TDMT_VND_TASK_PIPE_EXEC, "vnode-task-pipe-exec", SStreamTaskExecReq, SStreamTaskExecRsp) + TD_DEF_MSG_TYPE(TDMT_VND_TASK_MERGE_EXEC, "vnode-task-merge-exec", SStreamTaskExecReq, SStreamTaskExecRsp) + TD_DEF_MSG_TYPE(TDMT_VND_TASK_WRITE_EXEC, "vnode-task-write-exec", SStreamTaskExecReq, SStreamTaskExecRsp) + TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL) @@ -203,6 +209,8 @@ enum { TD_NEW_MSG_SEG(TDMT_SND_MSG) TD_DEF_MSG_TYPE(TDMT_SND_TASK_DEPLOY, "snode-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp) TD_DEF_MSG_TYPE(TDMT_SND_TASK_EXEC, "snode-task-exec", SStreamTaskExecReq, SStreamTaskExecRsp) + TD_DEF_MSG_TYPE(TDMT_SND_TASK_PIPE_EXEC, "snode-task-pipe-exec", SStreamTaskExecReq, SStreamTaskExecRsp) + TD_DEF_MSG_TYPE(TDMT_SND_TASK_MERGE_EXEC, "snode-task-merge-exec", SStreamTaskExecReq, SStreamTaskExecRsp) #if defined(TD_MSG_NUMBER_) TDMT_MAX diff --git a/include/common/trow.h b/include/common/trow.h index 4ae5a2277df649ba0d4a37985a4dd1965293ec3b..fc99cbc5b29506849d5a5ca272d83a79042f7739 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -166,6 +166,7 @@ typedef struct { #define TD_ROW_HEAD_LEN (sizeof(STSRow)) #define TD_ROW_NCOLS_LEN (sizeof(col_id_t)) +#define TD_ROW_INFO(r) ((r)->info) #define TD_ROW_TYPE(r) ((r)->type) #define TD_ROW_DELETE(r) ((r)->del) #define TD_ROW_ENDIAN(r) ((r)->endian) @@ -180,6 +181,7 @@ typedef struct { // (int32_t)ceil((double)nCols/TD_VTYPE_PARTS) should be added if TD_SUPPORT_BITMAP defined. #define TD_ROW_MAX_BYTES_FROM_SCHEMA(s) (schemaTLen(s) + TD_ROW_HEAD_LEN) +#define TD_ROW_SET_INFO(r, i) (TD_ROW_INFO(r) = (i)) #define TD_ROW_SET_TYPE(r, t) (TD_ROW_TYPE(r) = (t)) #define TD_ROW_SET_DELETE(r) (TD_ROW_DELETE(r) = 1) #define TD_ROW_SET_SVER(r, v) (TD_ROW_SVER(r) = (v)) @@ -473,6 +475,7 @@ static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { return terrno; } + TD_ROW_SET_INFO(pBuilder->pBuf, 0); TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType); uint32_t len = 0; diff --git a/include/common/ttypes.h b/include/common/ttypes.h index d7fcc28410872a56fab3ec2b62b02a22347cc9f6..59af14c22647a7013bcbf4117c9e737d902b3418 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -159,6 +159,7 @@ typedef struct { (IS_SIGNED_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP)) #define IS_CONVERT_AS_UNSIGNED(_t) (IS_UNSIGNED_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL)) +// TODO remove this function static FORCE_INLINE bool isNull(const void *val, int32_t type) { switch (type) { case TSDB_DATA_TYPE_BOOL: diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index eed13e031753495e8eb36e33a96d31f8048bc88f..5b88a9d6aff34bf3a151650f42c48e4ef325e81d 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -17,7 +17,9 @@ #define _TD_MND_H_ #include "monitor.h" +#include "tmsg.h" #include "tmsgcb.h" +#include "trpc.h" #ifdef __cplusplus extern "C" { diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index d04e9f817e7b601a22212765ba6612bb6ec649f2..a28c093e85bfbb7c5e1c3789e71b23e700b0549c 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -119,10 +119,11 @@ typedef enum { SDB_CONSUMER = 13, SDB_TOPIC = 14, SDB_VGROUP = 15, - SDB_STB = 16, - SDB_DB = 17, - SDB_FUNC = 18, - SDB_MAX = 19 + SDB_SMA = 16, + SDB_STB = 17, + SDB_DB = 18, + SDB_FUNC = 19, + SDB_MAX = 20 } ESdbType; typedef struct SSdb SSdb; diff --git a/include/libs/function/function.h b/include/libs/function/function.h index e33805437a81b2e96b9212c5563ef3f9944cb8cb..6c4774c3be60249960b6e633c705fa82d553d199 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -327,7 +327,7 @@ bool taosFillHasMoreResults(struct SFillInfo* pFillInfo); struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, - struct SFillColInfo* pFillCol, void* handle); + struct SFillColInfo* pFillCol, const char* id); void* taosDestroyFillInfo(struct SFillInfo *pFillInfo); int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, void** output, int32_t capacity); diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index ded12620d61cdde001ed0b2e37c32866fd4215a0..48a3a9bda8b200cab6c46520be64b017fcd98c00 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -199,6 +199,7 @@ typedef struct SIndexOptions { typedef struct SCreateIndexStmt { ENodeType type; EIndexType indexType; + bool ignoreExists; char indexName[TSDB_INDEX_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN]; SNodeList* pCols; @@ -207,6 +208,7 @@ typedef struct SCreateIndexStmt { typedef struct SDropIndexStmt { ENodeType type; + bool ignoreNotExists; char indexName[TSDB_INDEX_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN]; } SDropIndexStmt; diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 1ab8f2b48c6fd98a1f41ae0e5f6ae846ddb380d9..4c83a30bb9c5f55ffb3bf2ebfba2442019a19fd3 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -135,6 +135,7 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, QUERY_NODE_PHYSICAL_PLAN_SORT, QUERY_NODE_PHYSICAL_PLAN_INTERVAL, + QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW, QUERY_NODE_PHYSICAL_PLAN_DISPATCH, QUERY_NODE_PHYSICAL_PLAN_INSERT, QUERY_NODE_PHYSICAL_SUBPLAN, @@ -169,6 +170,7 @@ void nodesDestroyNode(SNodeptr pNode); SNodeList* nodesMakeList(); int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode); int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode); +int32_t nodesListMakeAppend(SNodeList** pList, SNodeptr pNode); int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc); int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc); SListCell* nodesListErase(SNodeList* pList, SListCell* pCell); diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 77c59a38c4444408096d33aa68c49bcd611cbd66..794e0ca85a3e2a8ebdce741504f8d844bbecd2a2 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -95,6 +95,7 @@ typedef struct SWindowLogicNode { int8_t intervalUnit; int8_t slidingUnit; SFillNode* pFill; + int64_t sessionGap; } SWindowLogicNode; typedef enum ESubplanType { @@ -110,7 +111,7 @@ typedef struct SSubplanId { int32_t subplanId; } SSubplanId; -typedef struct SSubLogicPlan { +typedef struct SLogicSubplan { ENodeType type; SSubplanId id; SNodeList* pChildren; @@ -120,7 +121,7 @@ typedef struct SSubLogicPlan { SVgroupsInfo* pVgroupList; int32_t level; int32_t splitFlag; -} SSubLogicPlan; +} SLogicSubplan; typedef struct SQueryLogicPlan { ENodeType type; @@ -213,10 +214,14 @@ typedef struct SExchangePhysiNode { SNodeList* pSrcEndPoints; // element is SDownstreamSource, scheduler fill by calling qSetSuplanExecutionNode } SExchangePhysiNode; -typedef struct SIntervalPhysiNode { +typedef struct SWinodwPhysiNode { SPhysiNode node; SNodeList* pExprs; // these are expression list of parameter expression of function SNodeList* pFuncs; +} SWinodwPhysiNode; + +typedef struct SIntervalPhysiNode { + SWinodwPhysiNode window; int64_t interval; int64_t offset; int64_t sliding; @@ -225,6 +230,11 @@ typedef struct SIntervalPhysiNode { SFillNode* pFill; } SIntervalPhysiNode; +typedef struct SSessionWinodwPhysiNode { + SWinodwPhysiNode window; + int64_t gap; +} SSessionWinodwPhysiNode; + typedef struct SDataSinkNode { ENodeType type; SDataBlockDescNode* pInputDataBlockDesc; diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index b6a7892f18b2e131173d00c0f11b3e168327ebdb..616e24d67d0b915c40532c4722f88283ae169990 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -191,8 +191,8 @@ typedef struct SStateWindowNode { typedef struct SSessionWindowNode { ENodeType type; // QUERY_NODE_SESSION_WINDOW - int64_t gap; // gap between two session window(in microseconds) SNode* pCol; + SNode* pGap; // gap between two session window(in microseconds) } SSessionWindowNode; typedef struct SIntervalWindowNode { diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 860c77de15ffdaa9c1678c2b1185c618cae52f0a..2254298e5c4d8dbe73f42760fac34f8df82a56a7 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -60,6 +60,8 @@ int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery); void qDestroyQuery(SQuery* pQueryNode); +int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); + #ifdef __cplusplus } #endif diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h new file mode 100644 index 0000000000000000000000000000000000000000..7200e78a418961100b2145f4859ddb2882cd8fa3 --- /dev/null +++ b/include/libs/stream/tstream.h @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdatablock.h" +#include "tmsg.h" +#include "tmsgcb.h" +#include "trpc.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef _TSTREAM_H_ +#define _TSTREAM_H_ + +enum { + STREAM_TASK_STATUS__RUNNING = 1, + STREAM_TASK_STATUS__STOP, +}; + +// pipe -> fetch/pipe queue +// merge -> merge queue +// write -> write queue +enum { + TASK_SINK_MSG__SND_PIPE = 1, + TASK_SINK_MSG__SND_MERGE, + TASK_SINK_MSG__VND_PIPE, + TASK_SINK_MSG__VND_MERGE, + TASK_SINK_MSG__VND_WRITE, +}; + +typedef struct { + int32_t nodeId; // 0 for snode + SEpSet epSet; +} SStreamTaskEp; + +typedef struct { + void* inputHandle; + void* executor; +} SStreamRunner; + +typedef struct { + int8_t parallelizable; + char* qmsg; + // followings are not applicable to encoder and decoder + int8_t numOfRunners; + SStreamRunner* runners; +} STaskExec; + +typedef struct { + int8_t reserved; +} STaskDispatcherInplace; + +typedef struct { + int32_t nodeId; + SEpSet epSet; +} STaskDispatcherFixedEp; + +typedef struct { + int8_t hashMethod; + SArray* info; +} STaskDispatcherShuffle; + +typedef struct { + int8_t reserved; + // not applicable to encoder and decoder + SHashObj* pHash; // groupId to tbuid +} STaskSinkTb; + +typedef struct { + int8_t reserved; +} STaskSinkSma; + +typedef struct { + int8_t reserved; +} STaskSinkFetch; + +typedef struct { + int8_t reserved; +} STaskSinkShow; + +enum { + TASK_SOURCE__SCAN = 1, + TASK_SOURCE__SINGLE, + TASK_SOURCE__MULTI, +}; + +enum { + TASK_EXEC__NONE = 1, + TASK_EXEC__EXEC, +}; + +enum { + TASK_DISPATCH__NONE = 1, + TASK_DISPATCH__INPLACE, + TASK_DISPATCH__FIXED, + TASK_DISPATCH__SHUFFLE, +}; + +enum { + TASK_SINK__NONE = 1, + TASK_SINK__TABLE, + TASK_SINK__SMA, + TASK_SINK__FETCH, + TASK_SINK__SHOW, +}; + +typedef struct { + int64_t streamId; + int32_t taskId; + int8_t status; + + int8_t sourceType; + int8_t execType; + int8_t sinkType; + int8_t dispatchType; + int16_t dispatchMsgType; + int32_t downstreamTaskId; + + // source preprocess + + // exec + STaskExec exec; + + // local sink + union { + STaskSinkTb tbSink; + STaskSinkSma smaSink; + STaskSinkFetch fetchSink; + STaskSinkShow showSink; + }; + + // dispatch + union { + STaskDispatcherInplace inplaceDispatcher; + STaskDispatcherFixedEp fixedEpDispatcher; + STaskDispatcherShuffle shuffleDispatcher; + }; + + // state storage + +} SStreamTask; + +SStreamTask* tNewSStreamTask(int64_t streamId); +int32_t tEncodeSStreamTask(SCoder* pEncoder, const SStreamTask* pTask); +int32_t tDecodeSStreamTask(SCoder* pDecoder, SStreamTask* pTask); +void tFreeSStreamTask(SStreamTask* pTask); + +typedef struct { + // SMsgHead head; + SStreamTask* task; +} SStreamTaskDeployReq; + +typedef struct { + int32_t reserved; +} SStreamTaskDeployRsp; + +typedef struct { + // SMsgHead head; + int64_t streamId; + int32_t taskId; + SArray* data; // SArray +} SStreamTaskExecReq; + +int32_t tEncodeSStreamTaskExecReq(void** buf, const SStreamTaskExecReq* pReq); +void* tDecodeSStreamTaskExecReq(const void* buf, SStreamTaskExecReq* pReq); +void tFreeSStreamTaskExecReq(SStreamTaskExecReq* pReq); + +typedef struct { + int32_t reserved; +} SStreamTaskExecRsp; + +typedef struct { + // SMsgHead head; + int64_t streamId; + int64_t version; + SArray* res; // SArray +} SStreamSinkReq; + +int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId); + +#ifdef __cplusplus +} +#endif + +#endif /* ifndef _TSTREAM_H_ */ diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index a38431a1b2b00553ed2cacefd260d05733299669..a2f88490f05f8b579b40f8209d2dab6b7ee934ce 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -35,6 +35,7 @@ typedef enum { TAOS_SYNC_STATE_FOLLOWER = 100, TAOS_SYNC_STATE_CANDIDATE = 101, TAOS_SYNC_STATE_LEADER = 102, + TAOS_SYNC_STATE_ERROR = 103, } ESyncState; typedef struct SSyncBuffer { @@ -68,17 +69,20 @@ typedef struct SSnapshot { typedef struct SSyncFSM { void* data; - // when value in pBuf finish a raft flow, FpCommitCb is called, code indicates the result + // when value in pMsg finish a raft flow, FpCommitCb is called, code indicates the result // user can do something according to the code and isWeak. for example, write data into tsdb - void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pBuf, SyncIndex index, bool isWeak, int32_t code); + void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state); - // when value in pBuf has been written into local log store, FpPreCommitCb is called, code indicates the result + // when value in pMsg has been written into local log store, FpPreCommitCb is called, code indicates the result // user can do something according to the code and isWeak. for example, write data into tsdb - void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pBuf, SyncIndex index, bool isWeak, int32_t code); + void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state); // when log entry is updated by a new one, FpRollBackCb is called // user can do something to roll back. for example, delete data from tsdb, or just ignore it - void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pBuf, SyncIndex index, bool isWeak, int32_t code); + void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state); // user should implement this function, use "data" to take snapshot into "snapshot" int32_t (*FpTakeSnapshot)(SSnapshot* snapshot); @@ -157,10 +161,14 @@ void syncCleanUp(); int64_t syncStart(const SSyncInfo* pSyncInfo); void syncStop(int64_t rid); int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg); -int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak); // use this function -int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak); // just for compatibility +int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak); ESyncState syncGetMyRole(int64_t rid); -void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole); + +// propose with sequence number, to implement linearizable semantics +int32_t syncPropose2(int64_t rid, const SRpcMsg* pMsg, bool isWeak, uint64_t seqNum); + +// for compatibility, the same as syncPropose +int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak); extern int32_t sDebugFlag; diff --git a/source/libs/transport/inc/rpcHead.h b/include/libs/transport/rpcHead.h similarity index 100% rename from source/libs/transport/inc/rpcHead.h rename to include/libs/transport/rpcHead.h diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 02029a996cd315bc5520da7041ccecee82f52045..6112a77a1279e56cd8fbeb5b9b8ef3c42d0ce849 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -83,7 +83,7 @@ typedef struct SRpcInit { typedef struct { void * val; int32_t len; - void (*free)(void *arg); + void (*freeFunc)(const void *arg); } SRpcCtxVal; typedef struct { diff --git a/include/os/osMemory.h b/include/os/osMemory.h index 6100419035ff9901bf6a6d5c3871496d0edcd533..62ac82782c5d6c77d27bea9df825fc8e5be69e60 100644 --- a/include/os/osMemory.h +++ b/include/os/osMemory.h @@ -20,12 +20,25 @@ extern "C" { #endif -#define tfree(x) \ - do { \ - if (x) { \ - free((void *)(x)); \ - (x) = 0; \ - } \ +// If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following sectio +#ifndef ALLOW_FORBID_FUNC + #define malloc MALLOC_FUNC_TAOS_FORBID + #define calloc CALLOC_FUNC_TAOS_FORBID + #define realloc REALLOC_FUNC_TAOS_FORBID + #define free FREE_FUNC_TAOS_FORBID +#endif + +void *taosMemoryMalloc(int32_t size); +void *taosMemoryCalloc(int32_t num, int32_t size); +void *taosMemoryRealloc(void *ptr, int32_t size); +void taosMemoryFree(const void *ptr); +int32_t taosMemorySize(void *ptr); + +#define taosMemoryFreeClear(ptr) \ + do { \ + taosMemoryFree(ptr); \ + (ptr)=NULL; \ } while (0) #ifdef __cplusplus diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 3c8cc8abd1bdc35f4dc634f534c0265239a6fac9..7af8dd37bffb9bad5e91abe21936e272442c2889 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -51,10 +51,31 @@ extern "C" { #endif -#if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) +#if defined(WINDOWS) #define htobe64 htonll #endif +#if defined(WINDOWS) + #define TAOS_EPOLL_WAIT_TIME 100 + typedef SOCKET eventfd_t; + #define eventfd(a, b) -1 + typedef SOCKET EpollFd; + #define EpollClose(pollFd) epoll_close(pollFd) + #ifndef EPOLLWAKEUP + #define EPOLLWAKEUP (1u << 29) + #endif +#elif defined(_TD_DARWIN_64) + #define TAOS_EPOLL_WAIT_TIME 500 + typedef int32_t SOCKET; + typedef SOCKET EpollFd; + #define EpollClose(pollFd) epoll_close(pollFd) +#else + #define TAOS_EPOLL_WAIT_TIME 500 + typedef int32_t SOCKET; + typedef SOCKET EpollFd; + #define EpollClose(pollFd) taosCloseSocket(pollFd) +#endif + #if defined(_TD_DARWIN_64) // #define htobe64 htonll @@ -64,12 +85,12 @@ extern "C" { # define htole16(x) OSSwapHostToLittleInt16(x) # define be16toh(x) OSSwapBigToHostInt16(x) # define le16toh(x) OSSwapLittleToHostInt16(x) - + # define htobe32(x) OSSwapHostToBigInt32(x) # define htole32(x) OSSwapHostToLittleInt32(x) # define be32toh(x) OSSwapBigToHostInt32(x) # define le32toh(x) OSSwapLittleToHostInt32(x) - + # define htobe64(x) OSSwapHostToBigInt64(x) # define htole64(x) OSSwapHostToLittleInt64(x) # define be64toh(x) OSSwapBigToHostInt64(x) @@ -83,6 +104,17 @@ extern "C" { #define TAOS_EPOLL_WAIT_TIME 500 +typedef int32_t SocketFd; +typedef SocketFd EpollFd; + +typedef struct TdSocket { +#if SOCKET_WITH_LOCK + TdThreadRwlock rwlock; +#endif + int refId; + SocketFd fd; +} *TdSocketPtr, TdSocket; + typedef struct TdSocketServer *TdSocketServerPtr; typedef struct TdSocket *TdSocketPtr; typedef struct TdEpoll *TdEpollPtr; @@ -91,6 +123,7 @@ int32_t taosSendto(TdSocketPtr pSocket, void * msg, int len, unsigned int flags, int32_t taosWriteSocket(TdSocketPtr pSocket, void *msg, int len); int32_t taosReadSocket(TdSocketPtr pSocket, void *msg, int len); int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, socklen_t *addrLen); +int32_t taosCloseSocketNoCheck1(SocketFd fd); int32_t taosCloseSocket(TdSocketPtr *ppSocket); int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer); int32_t taosShutDownSocketRD(TdSocketPtr pSocket); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 9e1e1c53dca9146249b72d9c83eb59862616dd04..87781b6313d73f3060afe90d6bba17197e8373bb 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -274,18 +274,23 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_STREAM_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03F1) #define TSDB_CODE_MND_INVALID_STREAM_OPTION TAOS_DEF_ERROR_CODE(0, 0x03F2) +// mnode-sma +#define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0400) +#define TSDB_CODE_MND_SMA_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0401) +#define TSDB_CODE_MND_INVALID_SMA_OPTION TAOS_DEF_ERROR_CODE(0, 0x0402) + // dnode -#define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400) -#define TSDB_CODE_DND_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0401) -#define TSDB_CODE_DND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0402) -#define TSDB_CODE_NODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0403) -#define TSDB_CODE_NODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0404) -#define TSDB_CODE_NODE_PARSE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0405) -#define TSDB_CODE_NODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0406) -#define TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0410) -#define TSDB_CODE_DND_VNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0411) -#define TSDB_CODE_DND_VNODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0412) -#define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0413) +#define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x04A0) +#define TSDB_CODE_DND_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x04A1) +#define TSDB_CODE_DND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x04A2) +#define TSDB_CODE_NODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x04A3) +#define TSDB_CODE_NODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x04A4) +#define TSDB_CODE_NODE_PARSE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x04A5) +#define TSDB_CODE_NODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x04A6) +#define TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x04A7) +#define TSDB_CODE_DND_VNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x04A8) +#define TSDB_CODE_DND_VNODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x04A9) +#define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x04AA) // vnode #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) @@ -309,7 +314,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_VND_IS_SYNCING TAOS_DEF_ERROR_CODE(0, 0x0513) #define TSDB_CODE_VND_INVALID_TSDB_STATE TAOS_DEF_ERROR_CODE(0, 0x0514) #define TSDB_CODE_VND_TB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0515) -#define TSDB_CODE_VND_HASH_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x0516) +#define TSDB_CODE_VND_SMA_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0516) +#define TSDB_CODE_VND_HASH_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x0517) // tsdb #define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) @@ -336,8 +342,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TDB_IVLD_TAG_VAL TAOS_DEF_ERROR_CODE(0, 0x0615) #define TSDB_CODE_TDB_NO_CACHE_LAST_ROW TAOS_DEF_ERROR_CODE(0, 0x0616) #define TSDB_CODE_TDB_TABLE_RECREATED TAOS_DEF_ERROR_CODE(0, 0x0617) -#define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META TAOS_DEF_ERROR_CODE(0, 0x0618) -#define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0619) +#define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0618) +#define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META TAOS_DEF_ERROR_CODE(0, 0x0619) +#define TSDB_CODE_TDB_INVALID_SMA_STAT TAOS_DEF_ERROR_CODE(0, 0x0620) // query #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) @@ -468,6 +475,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INVALID_PORT TAOS_DEF_ERROR_CODE(0, 0x2612) #define TSDB_CODE_PAR_INVALID_ENDPOINT TAOS_DEF_ERROR_CODE(0, 0x2613) #define TSDB_CODE_PAR_EXPRIE_STATEMENT TAOS_DEF_ERROR_CODE(0, 0x2614) +#define TSDB_CODE_PAR_INTERVAL_VALUE_TOO_SMALL TAOS_DEF_ERROR_CODE(0, 0x2615) #ifdef __cplusplus } diff --git a/include/util/tcoding.h b/include/util/tcoding.h index 943a1f9ecad450f33b4f0e0eb9ef28fae60bd813..838b175a287f3b1ae408f5be549a1fa8082f7b28 100644 --- a/include/util/tcoding.h +++ b/include/util/tcoding.h @@ -351,7 +351,7 @@ static FORCE_INLINE void *taosDecodeString(const void *buf, char **value) { uint64_t size = 0; buf = taosDecodeVariantU64(buf, &size); - *value = (char *)malloc((size_t)size + 1); + *value = (char *)taosMemoryMalloc((size_t)size + 1); if (*value == NULL) return NULL; memcpy(*value, buf, (size_t)size); @@ -386,7 +386,7 @@ static FORCE_INLINE int32_t taosEncodeBinary(void **buf, const void *value, int3 } static FORCE_INLINE void *taosDecodeBinary(const void *buf, void **value, int32_t valueLen) { - *value = malloc((size_t)valueLen); + *value = taosMemoryMalloc((size_t)valueLen); if (*value == NULL) return NULL; memcpy(*value, buf, (size_t)valueLen); diff --git a/include/util/tdef.h b/include/util/tdef.h index 7a3e26aa15a42cb3ddccea0b6673d2cdc774bf97..fd8194e63e63c2778bc33e4b9276aa52cf671cf3 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -453,6 +453,10 @@ enum { SND_WORKER_TYPE__UNIQUE, }; +#define MND_VGID -1 +#define QND_VGID 1 +#define VND_VGID 0 + #ifdef __cplusplus } #endif diff --git a/include/util/tencode.h b/include/util/tencode.h index 7e1b624dfb5095536a8c5f170b80af38afc68d52..cdde378b69f7954c168fb633b83139967b37545c 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -406,7 +406,7 @@ static FORCE_INLINE int32_t tDecodeBinaryAlloc(SCoder* pDecoder, void** val, uin if (tDecodeU64v(pDecoder, len) < 0) return -1; if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, *len)) return -1; - *val = malloc(*len); + *val = taosMemoryMalloc(*len); if (*val == NULL) return -1; memcpy(*val, TD_CODER_CURRENT(pDecoder), *len); diff --git a/include/util/tfreelist.h b/include/util/tfreelist.h index 0a507aeec9b35951e7e646559cf058d5cb05b69a..e9b5ca5fcaa0e26c45cd01d809ce9a4d7def42a9 100644 --- a/include/util/tfreelist.h +++ b/include/util/tfreelist.h @@ -31,7 +31,7 @@ typedef TD_SLIST(SFreeListNode) SFreeList; #define TFL_MALLOC(PTR, TYPE, SIZE, LIST) \ do { \ - void *ptr = malloc((SIZE) + sizeof(struct SFreeListNode)); \ + void *ptr = taosMemoryMalloc((SIZE) + sizeof(struct SFreeListNode)); \ if (ptr) { \ TD_SLIST_PUSH((LIST), (struct SFreeListNode *)ptr); \ ptr = ((struct SFreeListNode *)ptr)->payload; \ @@ -49,7 +49,7 @@ static FORCE_INLINE void tFreeListClear(SFreeList *pFL) { pNode = TD_SLIST_HEAD(pFL); if (pNode == NULL) break; TD_SLIST_POP(pFL); - free(pNode); + taosMemoryFree(pNode); } } diff --git a/include/util/tlist.h b/include/util/tlist.h index caa642491898243a79509338447452bcd6dd31fa..43833d7ecd84f09643546f3f3fa838edbd1dabf1 100644 --- a/include/util/tlist.h +++ b/include/util/tlist.h @@ -216,7 +216,7 @@ typedef struct { #define listNEles(l) TD_DLIST_NELES(l) #define listEleSize(l) ((l)->eleSize) #define isListEmpty(l) (TD_DLIST_NELES(l) == 0) -#define listNodeFree(n) free(n) +#define listNodeFree(n) taosMemoryFree(n) void tdListInit(SList *list, int32_t eleSize); void tdListEmpty(SList *list); diff --git a/include/util/types.h b/include/util/types.h index 981c457fc10598d7109c44845939364c428f7a66..d48995418e530646e8ef540bfb0630fc90cd9636 100644 --- a/include/util/types.h +++ b/include/util/types.h @@ -84,6 +84,8 @@ typedef uint16_t VarDataLenT; // maxVarDataLen: 32767 #define varDataLen(v) ((VarDataLenT *)(v))[0] #define varDataVal(v) ((char *)(v) + VARSTR_HEADER_SIZE) +#define NCHAR_WIDTH_TO_BYTES(n) ((n) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE) + typedef int32_t VarDataOffsetT; typedef struct tstr { diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index de6e72336d586874f85af0d3cc0760f04f554712..7d7e51bc27fa2fc858a6a346d110a345b060c73d 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -155,6 +155,7 @@ typedef struct SReqResultInfo { TAOS_FIELD* fields; uint32_t numOfCols; int32_t* length; + char** convertBuf; TAOS_ROW row; SResultColumn* pCol; uint32_t numOfRows; diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 6ebf9e71e002443065fcb8546d271b68846ed4e2..f328f518ac4b1dd8d923c83a2a99b818f81631dd 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -115,11 +115,11 @@ void destroyTscObj(void *pObj) { atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); taosThreadMutexDestroy(&pTscObj->mutex); - tfree(pTscObj); + taosMemoryFreeClear(pTscObj); } void *createTscObj(const char *user, const char *auth, const char *db, SAppInstInfo *pAppInfo) { - STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj)); + STscObj *pObj = (STscObj *)taosMemoryCalloc(1, sizeof(STscObj)); if (NULL == pObj) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; @@ -143,7 +143,7 @@ void *createTscObj(const char *user, const char *auth, const char *db, SAppInstI void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t type) { assert(pObj != NULL); - SRequestObj *pRequest = (SRequestObj *)calloc(1, sizeof(SRequestObj)); + SRequestObj *pRequest = (SRequestObj *)taosMemoryCalloc(1, sizeof(SRequestObj)); if (NULL == pRequest) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; @@ -156,7 +156,7 @@ void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t ty pRequest->type = type; pRequest->pTscObj = pObj; pRequest->body.fp = fp; // not used it yet - pRequest->msgBuf = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); + pRequest->msgBuf = taosMemoryCalloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); tsem_init(&pRequest->body.rspSem, 0, 0); registerRequest(pRequest); @@ -164,11 +164,18 @@ void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t ty } static void doFreeReqResultInfo(SReqResultInfo *pResInfo) { - tfree(pResInfo->pRspMsg); - tfree(pResInfo->length); - tfree(pResInfo->row); - tfree(pResInfo->pCol); - tfree(pResInfo->fields); + taosMemoryFreeClear(pResInfo->pRspMsg); + taosMemoryFreeClear(pResInfo->length); + taosMemoryFreeClear(pResInfo->row); + taosMemoryFreeClear(pResInfo->pCol); + taosMemoryFreeClear(pResInfo->fields); + + if (pResInfo->convertBuf != NULL) { + for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { + taosMemoryFreeClear(pResInfo->convertBuf[i]); + } + taosMemoryFreeClear(pResInfo->convertBuf); + } } static void doDestroyRequest(void *p) { @@ -177,10 +184,10 @@ static void doDestroyRequest(void *p) { assert(RID_VALID(pRequest->self)); - tfree(pRequest->msgBuf); - tfree(pRequest->sqlstr); - tfree(pRequest->pInfo); - tfree(pRequest->pDb); + taosMemoryFreeClear(pRequest->msgBuf); + taosMemoryFreeClear(pRequest->sqlstr); + taosMemoryFreeClear(pRequest->pInfo); + taosMemoryFreeClear(pRequest->pDb); doFreeReqResultInfo(&pRequest->body.resInfo); qDestroyQueryPlan(pRequest->body.pDag); @@ -190,7 +197,7 @@ static void doDestroyRequest(void *p) { } deregisterRequest(pRequest); - tfree(pRequest); + taosMemoryFreeClear(pRequest); } void destroyRequest(SRequestObj *pRequest) { @@ -349,7 +356,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { tscInfo("charset:%s is not valid in locale, charset remains:%s", charset, tsCharset); } - free(charset); + taosMemoryFree(charset); } else { // it may be windows system tscInfo("charset remains:%s", tsCharset); } diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 479281eec642e1e3b12581ab6a5bf5a12dc1fcc6..832a415efac6058340b22658e332bf871af7a3ec 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -166,7 +166,7 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { static int32_t hbAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code) { static int32_t emptyRspNum = 0; if (code != 0) { - tfree(param); + taosMemoryFreeClear(param); return -1; } @@ -179,12 +179,12 @@ static int32_t hbAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code) SAppInstInfo **pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); if (pInst == NULL || NULL == *pInst) { tscError("cluster not exist, key:%s", key); - tfree(param); + taosMemoryFreeClear(param); tFreeClientHbBatchRsp(&pRsp); return -1; } - tfree(param); + taosMemoryFreeClear(param); if (rspNum) { tscDebug("hb got %d rsp, %d empty rsp received before", rspNum, @@ -317,7 +317,7 @@ void hbFreeReq(void *req) { } SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { - SClientHbBatchReq *pBatchReq = calloc(1, sizeof(SClientHbBatchReq)); + SClientHbBatchReq *pBatchReq = taosMemoryCalloc(1, sizeof(SClientHbBatchReq)); if (pBatchReq == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; @@ -346,7 +346,7 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { if (code) { taosArrayDestroyEx(pBatchReq->reqs, hbFreeReq); - tfree(pBatchReq); + taosMemoryFreeClear(pBatchReq); } return pBatchReq; @@ -387,7 +387,7 @@ static void *hbThreadFunc(void *param) { continue; } int tlen = tSerializeSClientHbBatchReq(NULL, 0, pReq); - void *buf = malloc(tlen); + void *buf = taosMemoryMalloc(tlen); if (buf == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; tFreeClientHbBatchReq(pReq, false); @@ -396,13 +396,13 @@ static void *hbThreadFunc(void *param) { } tSerializeSClientHbBatchReq(buf, tlen, pReq); - SMsgSendInfo *pInfo = calloc(1, sizeof(SMsgSendInfo)); + SMsgSendInfo *pInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (pInfo == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; tFreeClientHbBatchReq(pReq, false); hbClearReqInfo(pAppHbMgr); - free(buf); + taosMemoryFree(buf); break; } pInfo->fp = hbAsyncCallBack; @@ -458,7 +458,7 @@ static void hbStopThread() { SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { hbMgrInit(); - SAppHbMgr *pAppHbMgr = malloc(sizeof(SAppHbMgr)); + SAppHbMgr *pAppHbMgr = taosMemoryMalloc(sizeof(SAppHbMgr)); if (pAppHbMgr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -478,7 +478,7 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { if (pAppHbMgr->activeInfo == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - free(pAppHbMgr); + taosMemoryFree(pAppHbMgr); return NULL; } @@ -488,7 +488,7 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { if (pAppHbMgr->connInfo == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - free(pAppHbMgr); + taosMemoryFree(pAppHbMgr); return NULL; } @@ -580,7 +580,7 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int32_t connId, int64_t clusterId, int3 switch (hbType) { case HEARTBEAT_TYPE_QUERY: { - int64_t *pClusterId = malloc(sizeof(int64_t)); + int64_t *pClusterId = taosMemoryMalloc(sizeof(int64_t)); *pClusterId = clusterId; info.param = pClusterId; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index b6bb14e9a4a5c1a6298ba22751651f51f859b6f0..0fa1391b6e9de069faf12d9838cae358fdac06d8 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -100,7 +100,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); SAppInstInfo* p = NULL; if (pInst == NULL) { - p = calloc(1, sizeof(struct SAppInstInfo)); + p = taosMemoryCalloc(1, sizeof(struct SAppInstInfo)); p->mgmtEp = epSet; p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores); p->pAppHbMgr = appHbMgrInit(p, key); @@ -111,7 +111,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, taosThreadMutexUnlock(&appInfo.mutex); - tfree(key); + taosMemoryFreeClear(key); return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst); } @@ -122,7 +122,7 @@ int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj* return TSDB_CODE_TSC_OUT_OF_MEMORY; } - (*pRequest)->sqlstr = malloc(sqlLen + 1); + (*pRequest)->sqlstr = taosMemoryMalloc(sqlLen + 1); if ((*pRequest)->sqlstr == NULL) { tscError("0x%" PRIx64 " failed to prepare sql string buffer", (*pRequest)->self); (*pRequest)->msgBuf = strdup("failed to prepare sql string buffer"); @@ -212,7 +212,7 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t assert(pSchema != NULL && numOfCols > 0); pResInfo->numOfCols = numOfCols; - pResInfo->fields = calloc(numOfCols, sizeof(pSchema[0])); + pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(pSchema[0])); for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { pResInfo->fields[i].bytes = pSchema[i].bytes; @@ -421,7 +421,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t } static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { - SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (pMsgSendInfo == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; @@ -441,14 +441,14 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { if (db != NULL) { tstrncpy(connectReq.db, db, sizeof(connectReq.db)); } - tfree(db); + taosMemoryFreeClear(db); connectReq.pid = htonl(appInfo.pid); connectReq.startTime = htobe64(appInfo.startTime); tstrncpy(connectReq.app, appInfo.appName, sizeof(connectReq.app)); int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq); - void* pReq = malloc(contLen); + void* pReq = taosMemoryMalloc(contLen); tSerializeSConnectReq(pReq, contLen, &connectReq); pMsgSendInfo->msgInfo.len = contLen; @@ -458,8 +458,8 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { assert(pMsgBody != NULL); - tfree(pMsgBody->msgInfo.pData); - tfree(pMsgBody); + taosMemoryFreeClear(pMsgBody->msgInfo.pData); + taosMemoryFreeClear(pMsgBody); } bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType) { return msgType == TDMT_VND_QUERY_RSP || msgType == TDMT_VND_FETCH_RSP || msgType == TDMT_VND_RES_READY_RSP || msgType == TDMT_VND_QUERY_HEARTBEAT_RSP; @@ -500,7 +500,7 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { SDataBuf buf = {.len = pMsg->contLen, .pData = NULL, .handle = pMsg->handle}; if (pMsg->contLen > 0) { - buf.pData = calloc(1, pMsg->contLen); + buf.pData = taosMemoryCalloc(1, pMsg->contLen); if (buf.pData == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; pMsg->code = TSDB_CODE_OUT_OF_MEMORY; @@ -592,7 +592,7 @@ void* doFetchRow(SRequestObj* pRequest) { } SVgroupInfo* pVgroupInfo = taosArrayGet(pShowReqInfo->pArray, pShowReqInfo->currentIndex); - SVShowTablesReq* pShowReq = calloc(1, sizeof(SVShowTablesReq)); + SVShowTablesReq* pShowReq = taosMemoryCalloc(1, sizeof(SVShowTablesReq)); pShowReq->head.vgId = htonl(pVgroupInfo->vgId); pRequest->body.requestMsg.len = sizeof(SVShowTablesReq); @@ -634,18 +634,30 @@ _return: for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) { SResultColumn* pCol = &pResultInfo->pCol[i]; - if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { + int32_t type = pResultInfo->fields[i].type; + int32_t bytes = pResultInfo->fields[i].bytes; + + if (IS_VAR_DATA_TYPE(type)) { if (pCol->offset[pResultInfo->current] != -1) { char* pStart = pResultInfo->pCol[i].offset[pResultInfo->current] + pResultInfo->pCol[i].pData; pResultInfo->length[i] = varDataLen(pStart); pResultInfo->row[i] = varDataVal(pStart); + + if (type == TSDB_DATA_TYPE_NCHAR) { + int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(pStart), varDataLen(pStart), varDataVal(pResultInfo->convertBuf[i])); + ASSERT(len <= bytes); + + pResultInfo->row[i] = varDataVal(pResultInfo->convertBuf[i]); + varDataSetLen(pResultInfo->convertBuf[i], len); + pResultInfo->length[i] = len; + } } else { pResultInfo->row[i] = NULL; } } else { if (!colDataIsNull_f(pCol->nullbitmap, pResultInfo->current)) { - pResultInfo->row[i] = pResultInfo->pCol[i].pData + pResultInfo->fields[i].bytes * pResultInfo->current; + pResultInfo->row[i] = pResultInfo->pCol[i].pData + bytes * pResultInfo->current; } else { pResultInfo->row[i] = NULL; } @@ -658,16 +670,23 @@ _return: static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) { if (pResInfo->row == NULL) { - pResInfo->row = calloc(pResInfo->numOfCols, POINTER_BYTES); - pResInfo->pCol = calloc(pResInfo->numOfCols, sizeof(SResultColumn)); - pResInfo->length = calloc(pResInfo->numOfCols, sizeof(int32_t)); - } + pResInfo->row = taosMemoryCalloc(pResInfo->numOfCols, POINTER_BYTES); + pResInfo->pCol = taosMemoryCalloc(pResInfo->numOfCols, sizeof(SResultColumn)); + pResInfo->length = taosMemoryCalloc(pResInfo->numOfCols, sizeof(int32_t)); + pResInfo->convertBuf = taosMemoryCalloc(pResInfo->numOfCols, POINTER_BYTES); - if (pResInfo->row == NULL || pResInfo->pCol == NULL || pResInfo->length == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } else { - return TSDB_CODE_SUCCESS; + if (pResInfo->row == NULL || pResInfo->pCol == NULL || pResInfo->length == NULL || pResInfo->convertBuf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for(int32_t i = 0; i < pResInfo->numOfCols; ++i) { + if(pResInfo->fields[i].type == TSDB_DATA_TYPE_NCHAR) { + pResInfo->convertBuf[i] = taosMemoryCalloc(1, NCHAR_WIDTH_TO_BYTES(pResInfo->fields[i].bytes)); + } + } } + + return TSDB_CODE_SUCCESS; } int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows) { diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 9c2b23d7407e7ab249f11f739c6b147e55456a26..43143917436435d735a9cdc733902e60f20457a0 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -32,7 +32,7 @@ int32_t genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; setErrno(pRequest, code); - free(pMsg->pData); + taosMemoryFree(pMsg->pData); tsem_post(&pRequest->body.rspSem); return code; } @@ -40,7 +40,7 @@ int32_t genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code) { int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; if (code != TSDB_CODE_SUCCESS) { - free(pMsg->pData); + taosMemoryFree(pMsg->pData); setErrno(pRequest, code); tsem_post(&pRequest->body.rspSem); return code; @@ -77,13 +77,13 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, connectRsp.clusterId, pTscObj->pAppInfo->numOfConns); - free(pMsg->pData); + taosMemoryFree(pMsg->pData); tsem_post(&pRequest->body.rspSem); return 0; } SMsgSendInfo* buildMsgInfoImpl(SRequestObj *pRequest) { - SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); pMsgSendInfo->requestObjRefId = pRequest->self; pMsgSendInfo->requestId = pRequest->requestId; @@ -96,13 +96,13 @@ SMsgSendInfo* buildMsgInfoImpl(SRequestObj *pRequest) { retrieveReq.showId = pRequest->body.showInfo.execId; int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &retrieveReq); - void* pReq = malloc(contLen); + void* pReq = taosMemoryMalloc(contLen); tSerializeSRetrieveTableReq(pReq, contLen, &retrieveReq); pMsgSendInfo->msgInfo.pData = pReq; pMsgSendInfo->msgInfo.len = contLen; pMsgSendInfo->msgInfo.handle = NULL; } else { - SVShowTablesFetchReq* pFetchMsg = calloc(1, sizeof(SVShowTablesFetchReq)); + SVShowTablesFetchReq* pFetchMsg = taosMemoryCalloc(1, sizeof(SVShowTablesFetchReq)); if (pFetchMsg == NULL) { return NULL; } @@ -135,12 +135,12 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) { tDeserializeSShowRsp(pMsg->pData, pMsg->len, &showRsp); STableMetaRsp *pMetaMsg = &showRsp.tableMeta; - tfree(pRequest->body.resInfo.pRspMsg); + taosMemoryFreeClear(pRequest->body.resInfo.pRspMsg); pRequest->body.resInfo.pRspMsg = pMsg->pData; SReqResultInfo* pResInfo = &pRequest->body.resInfo; if (pResInfo->fields == NULL) { - TAOS_FIELD* pFields = calloc(pMetaMsg->numOfColumns, sizeof(TAOS_FIELD)); + TAOS_FIELD* pFields = taosMemoryCalloc(pMetaMsg->numOfColumns, sizeof(TAOS_FIELD)); for (int32_t i = 0; i < pMetaMsg->numOfColumns; ++i) { SSchema* pSchema = &pMetaMsg->pSchemas[i]; tstrncpy(pFields[i].name, pSchema->name, tListLen(pFields[i].name)); @@ -171,7 +171,7 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) { int32_t processRetrieveMnodeRsp(void* param, const SDataBuf* pMsg, int32_t code) { SRequestObj *pRequest = param; SReqResultInfo *pResInfo = &pRequest->body.resInfo; - tfree(pResInfo->pRspMsg); + taosMemoryFreeClear(pResInfo->pRspMsg); if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); @@ -204,7 +204,7 @@ int32_t processRetrieveVndRsp(void* param, const SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; SReqResultInfo* pResInfo = &pRequest->body.resInfo; - tfree(pResInfo->pRspMsg); + taosMemoryFreeClear(pResInfo->pRspMsg); if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); @@ -237,7 +237,7 @@ int32_t processRetrieveVndRsp(void* param, const SDataBuf* pMsg, int32_t code) { int32_t processCreateDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { // todo rsp with the vnode id list SRequestObj* pRequest = param; - free(pMsg->pData); + taosMemoryFree(pMsg->pData); if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); } @@ -266,7 +266,7 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { } if (code != TSDB_CODE_SUCCESS) { - free(pMsg->pData); + taosMemoryFree(pMsg->pData); setErrno(pRequest, code); tsem_post(&pRequest->body.rspSem); return code; @@ -284,7 +284,7 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { if (code != 0) { terrno = code; if (output.dbVgroup) taosHashCleanup(output.dbVgroup->vgHash); - tfree(output.dbVgroup); + taosMemoryFreeClear(output.dbVgroup); tscError("failed to build use db output since %s", terrstr()); } else { @@ -304,7 +304,7 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { tNameGetDbName(&name, db); setConnectionDB(pRequest->pTscObj, db); - free(pMsg->pData); + taosMemoryFree(pMsg->pData); tsem_post(&pRequest->body.rspSem); return 0; } @@ -313,7 +313,7 @@ int32_t processCreateTableRsp(void* param, const SDataBuf* pMsg, int32_t code) { assert(pMsg != NULL && param != NULL); SRequestObj* pRequest = param; - free(pMsg->pData); + taosMemoryFree(pMsg->pData); if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); tsem_post(&pRequest->body.rspSem); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index d05ad06a66666f576d68771d0b3cf7686f85111a..52d5400b0bffe8c6f352a8bd2731cccdbb8f13a0 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -136,14 +136,14 @@ typedef struct { } SMqCommitCbParam; tmq_conf_t* tmq_conf_new() { - tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t)); + tmq_conf_t* conf = taosMemoryCalloc(1, sizeof(tmq_conf_t)); conf->auto_commit = false; conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST; return conf; } void tmq_conf_destroy(tmq_conf_t* conf) { - if (conf) free(conf); + if (conf) taosMemoryFree(conf); } tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { @@ -184,7 +184,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } tmq_list_t* tmq_list_new() { - tmq_list_t* ptr = malloc(sizeof(tmq_list_t) + 8 * sizeof(char*)); + tmq_list_t* ptr = taosMemoryMalloc(sizeof(tmq_list_t) + 8 * sizeof(char*)); if (ptr == NULL) { return ptr; } @@ -254,7 +254,7 @@ tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq) { } tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen) { - tmq_t* pTmq = calloc(sizeof(tmq_t), 1); + tmq_t* pTmq = taosMemoryCalloc(sizeof(tmq_t), 1); if (pTmq == NULL) { return NULL; } @@ -317,7 +317,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); tEncodeSMqCMCommitOffsetReq(&encoder, &req); int32_t tlen = encoder.pos; - void* buf = malloc(tlen); + void* buf = taosMemoryMalloc(tlen); if (buf == NULL) { tCoderClear(&encoder); return -1; @@ -333,7 +333,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in tscError("failed to malloc request"); } - SMqCommitCbParam* pParam = malloc(sizeof(SMqCommitCbParam)); + SMqCommitCbParam* pParam = taosMemoryMalloc(sizeof(SMqCommitCbParam)); if (pParam == NULL) { return -1; } @@ -361,7 +361,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in } tsem_destroy(&pParam->rspSem); - free(pParam); + taosMemoryFree(pParam); if (pArray) { taosArrayDestroy(pArray); @@ -394,7 +394,7 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { tNameSetDbName(&name, tmq->pTscObj->acctId, dbName, strlen(dbName)); tNameFromString(&name, topicName, T_NAME_TABLE); - char* topicFname = calloc(1, TSDB_TOPIC_FNAME_LEN); + char* topicFname = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN); if (topicFname == NULL) { goto _return; } @@ -405,11 +405,11 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { topic.vgs = taosArrayInit(0, sizeof(SMqClientVg)); taosArrayPush(tmq->clientTopics, &topic); taosArrayPush(req.topicNames, &topicFname); - free(dbName); + taosMemoryFree(dbName); } int tlen = tSerializeSCMSubscribeReq(NULL, &req); - void* buf = malloc(tlen); + void* buf = taosMemoryMalloc(tlen); if (buf == NULL) { goto _return; } @@ -505,10 +505,10 @@ TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbNa .sql = (char*)sql, }; tNameExtractFullName(&name, req.name); - strcpy(req.outputTbName, tbName); + strcpy(req.outputSTbName, tbName); int tlen = tSerializeSCMCreateStreamReq(NULL, 0, &req); - void* buf = malloc(tlen); + void* buf = taosMemoryMalloc(tlen); if (buf == NULL) { goto _return; } @@ -532,7 +532,7 @@ TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbNa tsem_wait(&pRequest->body.rspSem); _return: - tfree(astStr); + taosMemoryFreeClear(astStr); qDestroyQuery(pQueryNode); /*if (sendInfo != NULL) {*/ /*destroySendMsgInfo(sendInfo);*/ @@ -594,7 +594,7 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i tNameExtractFullName(&name, req.name); int tlen = tSerializeSCMCreateTopicReq(NULL, 0, &req); - void* buf = malloc(tlen); + void* buf = taosMemoryMalloc(tlen); if (buf == NULL) { goto _return; } @@ -618,7 +618,7 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i tsem_wait(&pRequest->body.rspSem); _return: - tfree(astStr); + taosMemoryFreeClear(astStr); qDestroyQuery(pQueryNode); /*if (sendInfo != NULL) {*/ /*destroySendMsgInfo(sendInfo);*/ @@ -757,7 +757,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { #if 0 if (pParam->sync == 1) { - /**pParam->msg = malloc(sizeof(tmq_message_t));*/ + /**pParam->msg = taosMemoryMalloc(sizeof(tmq_message_t));*/ *pParam->msg = taosAllocateQitem(sizeof(tmq_message_t)); if (*pParam->msg) { memcpy(*pParam->msg, pMsg->pData, sizeof(SMqRspHead)); @@ -774,7 +774,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { } #endif - /*SMqConsumeRsp* pRsp = calloc(1, sizeof(SMqConsumeRsp));*/ + /*SMqConsumeRsp* pRsp = taosMemoryCalloc(1, sizeof(SMqConsumeRsp));*/ tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t)); if (pRsp == NULL) { goto WRITE_QUEUE_FAIL; @@ -884,7 +884,7 @@ END: int32_t tmqAskEp(tmq_t* tmq, bool sync) { int32_t tlen = sizeof(SMqCMGetSubEpReq); - SMqCMGetSubEpReq* req = malloc(tlen); + SMqCMGetSubEpReq* req = taosMemoryMalloc(tlen); if (req == NULL) { tscError("failed to malloc get subscribe ep buf"); return -1; @@ -893,21 +893,21 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) { req->epoch = htonl(tmq->epoch); strcpy(req->cgroup, tmq->groupId); - SMqAskEpCbParam* pParam = malloc(sizeof(SMqAskEpCbParam)); + SMqAskEpCbParam* pParam = taosMemoryMalloc(sizeof(SMqAskEpCbParam)); if (pParam == NULL) { tscError("failed to malloc subscribe param"); - free(req); + taosMemoryFree(req); return -1; } pParam->tmq = tmq; pParam->sync = sync; tsem_init(&pParam->rspSem, 0, 0); - SMsgSendInfo* sendInfo = malloc(sizeof(SMsgSendInfo)); + SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo)); if (sendInfo == NULL) { tsem_destroy(&pParam->rspSem); - free(pParam); - free(req); + taosMemoryFree(pParam); + taosMemoryFree(req); return -1; } @@ -967,7 +967,7 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTo reqOffset = tmq->resetOffsetCfg; } - SMqPollReq* pReq = malloc(sizeof(SMqPollReq)); + SMqPollReq* pReq = taosMemoryMalloc(sizeof(SMqPollReq)); if (pReq == NULL) { return NULL; } @@ -1003,7 +1003,7 @@ tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) { return NULL; } - SMqPollCbParam* pParam = malloc(sizeof(SMqPollCbParam)); + SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam)); if (pParam == NULL) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); // TODO: out of mem @@ -1016,7 +1016,7 @@ tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) { pParam->msg = &msg; tsem_init(&pParam->rspSem, 0, 0); - SMsgSendInfo* sendInfo = malloc(sizeof(SMsgSendInfo)); + SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo)); if (sendInfo == NULL) { return NULL; } @@ -1071,9 +1071,9 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { tsem_post(&tmq->rspSem); return -1; } - SMqPollCbParam* pParam = malloc(sizeof(SMqPollCbParam)); + SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam)); if (pParam == NULL) { - free(pReq); + taosMemoryFree(pReq); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); tsem_post(&tmq->rspSem); return -1; @@ -1083,10 +1083,10 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { pParam->epoch = tmq->epoch; pParam->sync = 0; - SMsgSendInfo* sendInfo = malloc(sizeof(SMsgSendInfo)); + SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo)); if (sendInfo == NULL) { - free(pReq); - free(pParam); + taosMemoryFree(pReq); + taosMemoryFree(pParam); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); tsem_post(&tmq->rspSem); return -1; @@ -1258,7 +1258,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { return NULL; } - SMqPollCbParam* param = malloc(sizeof(SMqPollCbParam)); + SMqPollCbParam* param = taosMemoryMalloc(sizeof(SMqPollCbParam)); if (param == NULL) { ASSERT(false); taosMsleep(blocking_time); @@ -1289,7 +1289,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { tsem_wait(¶m->rspSem); tsem_destroy(¶m->rspSem); - free(param); + taosMemoryFree(param); if (tmq_message == NULL) { if (beginVgIdx == pTopic->nextVgIdx) { @@ -1331,7 +1331,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)}; - SMqCommitCbParam* pParam = malloc(sizeof(SMqCommitCbParam)); + SMqCommitCbParam* pParam = taosMemoryMalloc(sizeof(SMqCommitCbParam)); if (pParam == NULL) { continue; } @@ -1360,7 +1360,7 @@ void tmq_message_destroy(tmq_message_t* tmq_message) { if (tmq_message == NULL) return; SMqPollRsp* pRsp = &tmq_message->msg; tDeleteSMqConsumeRsp(pRsp); - /*free(tmq_message);*/ + /*taosMemoryFree(tmq_message);*/ taosFreeQitem(tmq_message); } @@ -1403,7 +1403,7 @@ char* tmq_get_topic_name(tmq_message_t* message) { return "not implemented yet"; #if 0 tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) { - tmq_t* pTmq = malloc(sizeof(tmq_t)); + tmq_t* pTmq = taosMemoryMalloc(sizeof(tmq_t)); if (pTmq == NULL) { return NULL; } @@ -1417,7 +1417,7 @@ tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) { static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { assert(pMsgBody != NULL); - tfree(pMsgBody->msgInfo.pData); - tfree(pMsgBody); + taosMemoryFreeClear(pMsgBody->msgInfo.pData); + taosMemoryFreeClear(pMsgBody); } #endif diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index ac092c8f101b808956f3908a8a839852630d3176..629dd90da3b531f373ff353ea6bd4123215f65aa 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -52,7 +52,7 @@ TEST(testCase, driverInit_Test) { // taosInitGlobalCfg(); // taos_init(); } - +#if 0 TEST(testCase, connect_Test) { // taos_options(TSDB_OPTION_CONFIGDIR, "/home/ubuntu/first/cfg"); @@ -652,6 +652,7 @@ TEST(testCase, projection_query_stables) { taos_free_result(pRes); taos_close(pConn); } +#endif TEST(testCase, agg_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -660,7 +661,7 @@ TEST(testCase, agg_query_tables) { TAOS_RES* pRes = taos_query(pConn, "use abc1"); taos_free_result(pRes); - pRes = taos_query(pConn, "select count(*), sum(k),min(k),max(k) from tu"); + pRes = taos_query(pConn, "select count(*) from tu"); if (taos_errno(pRes) != 0) { printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 67e7333597a6f805d132a338c78b25ac28091a47..2aeaad5a38fabfe45f51791acc1bcee81e312c09 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -115,7 +115,7 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con newSize = newSize * 1.5; } - char* buf = realloc(pColumnInfoData->pData, newSize); + char* buf = taosMemoryRealloc(pColumnInfoData->pData, newSize); if (buf == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -178,7 +178,7 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c uint32_t total = numOfRow1 + numOfRow2; if (BitmapLen(numOfRow1) < BitmapLen(total)) { - char* tmp = realloc(pColumnInfoData->nullbitmap, BitmapLen(total)); + char* tmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(total)); uint32_t extend = BitmapLen(total) - BitmapLen(numOfRow1); memset(tmp + BitmapLen(numOfRow1), 0, extend); pColumnInfoData->nullbitmap = tmp; @@ -218,7 +218,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { // Handle the bitmap - char* p = realloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2)); + char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2)); if (p == NULL) { // TODO } @@ -232,7 +232,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co uint32_t len = pSource->varmeta.length; uint32_t oldLen = pColumnInfoData->varmeta.length; if (pColumnInfoData->varmeta.allocLen < len + oldLen) { - char* tmp = realloc(pColumnInfoData->pData, len + oldLen); + char* tmp = taosMemoryRealloc(pColumnInfoData->pData, len + oldLen); if (tmp == NULL) { return TSDB_CODE_VND_OUT_OF_MEMORY; } @@ -247,7 +247,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co doBitmapMerge(pColumnInfoData, numOfRow1, pSource, numOfRow2); int32_t newSize = (numOfRow1 + numOfRow2) * pColumnInfoData->info.bytes; - char* tmp = realloc(pColumnInfoData->pData, newSize); + char* tmp = taosMemoryRealloc(pColumnInfoData->pData, newSize); if (tmp == NULL) { return TSDB_CODE_VND_OUT_OF_MEMORY; } @@ -268,16 +268,16 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { // Handle the bitmap - char* p = realloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * numOfRows); + char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * numOfRows); if (p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - pColumnInfoData->varmeta.offset = (int32_t*) p; + pColumnInfoData->varmeta.offset = (int32_t*)p; memcpy(pColumnInfoData->varmeta.offset, pSource->varmeta.offset, sizeof(int32_t) * numOfRows); if (pColumnInfoData->varmeta.allocLen < pSource->varmeta.length) { - char* tmp = realloc(pColumnInfoData->pData, pSource->varmeta.length); + char* tmp = taosMemoryRealloc(pColumnInfoData->pData, pSource->varmeta.length); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -289,7 +289,7 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length); pColumnInfoData->varmeta.length = pSource->varmeta.length; } else { - char* tmp = realloc(pColumnInfoData->nullbitmap, BitmapLen(numOfRows)); + char* tmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(numOfRows)); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -298,7 +298,7 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p memcpy(pColumnInfoData->nullbitmap, pSource->nullbitmap, BitmapLen(numOfRows)); int32_t newSize = numOfRows * pColumnInfoData->info.bytes; - tmp = realloc(pColumnInfoData->pData, newSize); + tmp = taosMemoryRealloc(pColumnInfoData->pData, newSize); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -349,7 +349,7 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) { uint32_t newLen = colDataGetLength(pCol1, pSrc->info.rows); int32_t newSize = oldLen + newLen; - char* tmp = realloc(pCol2->pData, newSize); + char* tmp = taosMemoryRealloc(pCol2->pData, newSize); if (tmp != NULL) { pCol2->pData = tmp; colDataMergeCol(pCol2, pDest->info.rows, pCol1, pSrc->info.rows); @@ -453,7 +453,7 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 return NULL; } - SSDataBlock* pDst = calloc(1, sizeof(SSDataBlock)); + SSDataBlock* pDst = taosMemoryCalloc(1, sizeof(SSDataBlock)); if (pDst == NULL) { return NULL; } @@ -470,10 +470,10 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 if (IS_VAR_DATA_TYPE(pSrcCol->info.type)) { SVarColAttr* pAttr = &colInfo.varmeta; - pAttr->offset = calloc(rowCount, sizeof(int32_t)); + pAttr->offset = taosMemoryCalloc(rowCount, sizeof(int32_t)); } else { - colInfo.nullbitmap = calloc(1, BitmapLen(rowCount)); - colInfo.pData = calloc(rowCount, colInfo.info.bytes); + colInfo.nullbitmap = taosMemoryCalloc(1, BitmapLen(rowCount)); + colInfo.pData = taosMemoryCalloc(rowCount, colInfo.info.bytes); } taosArrayPush(pDst->pDataBlock, &colInfo); @@ -562,7 +562,7 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { if (IS_VAR_DATA_TYPE(pCol->info.type)) { if (pCol->varmeta.allocLen < colLength) { - char* tmp = realloc(pCol->pData, colLength); + char* tmp = taosMemoryRealloc(pCol->pData, colLength); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -587,11 +587,11 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) { if (pBlock->info.rowSize == 0) { size_t rowSize = 0; - size_t numOfCols = pBlock->info.numOfCols; - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); - rowSize += pColInfo->info.bytes; - } + size_t numOfCols = pBlock->info.numOfCols; + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); + rowSize += pColInfo->info.bytes; + } pBlock->info.rowSize = rowSize; } @@ -610,7 +610,7 @@ size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock) { } SSchema* blockDataExtractSchema(const SSDataBlock* pBlock, int32_t* numOfCols) { - SSchema* pSchema = calloc(pBlock->info.numOfCols, sizeof(SSchema)); + SSchema* pSchema = taosMemoryCalloc(pBlock->info.numOfCols, sizeof(SSchema)); for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); pSchema[i].bytes = pColInfoData->info.bytes; @@ -637,7 +637,7 @@ double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { if (IS_VAR_DATA_TYPE(pColInfo->info.type)) { rowSize += sizeof(int32_t); } else { - rowSize += 1/8.0; // one bit for each record + rowSize += 1 / 8.0; // one bit for each record } } @@ -819,7 +819,7 @@ static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) { int32_t rows = pDataBlock->info.rows; int32_t numOfCols = pDataBlock->info.numOfCols; - SColumnInfoData* pCols = calloc(numOfCols, sizeof(SColumnInfoData)); + SColumnInfoData* pCols = taosMemoryCalloc(numOfCols, sizeof(SColumnInfoData)); if (pCols == NULL) { return NULL; } @@ -829,14 +829,14 @@ static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) { pCols[i].info = pColInfoData->info; if (IS_VAR_DATA_TYPE(pCols[i].info.type)) { - pCols[i].varmeta.offset = calloc(rows, sizeof(int32_t)); - pCols[i].pData = calloc(1, pColInfoData->varmeta.length); + pCols[i].varmeta.offset = taosMemoryCalloc(rows, sizeof(int32_t)); + pCols[i].pData = taosMemoryCalloc(1, pColInfoData->varmeta.length); pCols[i].varmeta.length = pColInfoData->varmeta.length; pCols[i].varmeta.allocLen = pCols[i].varmeta.length; } else { - pCols[i].nullbitmap = calloc(1, BitmapLen(rows)); - pCols[i].pData = calloc(rows, pCols[i].info.bytes); + pCols[i].nullbitmap = taosMemoryCalloc(1, BitmapLen(rows)); + pCols[i].pData = taosMemoryCalloc(rows, pCols[i].info.bytes); } } @@ -851,22 +851,22 @@ static void copyBackToBlock(SSDataBlock* pDataBlock, SColumnInfoData* pCols) { pColInfoData->info = pCols[i].info; if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - tfree(pColInfoData->varmeta.offset); + taosMemoryFreeClear(pColInfoData->varmeta.offset); pColInfoData->varmeta = pCols[i].varmeta; } else { - tfree(pColInfoData->nullbitmap); + taosMemoryFreeClear(pColInfoData->nullbitmap); pColInfoData->nullbitmap = pCols[i].nullbitmap; } - tfree(pColInfoData->pData); + taosMemoryFreeClear(pColInfoData->pData); pColInfoData->pData = pCols[i].pData; } - tfree(pCols); + taosMemoryFreeClear(pCols); } static int32_t* createTupleIndex(size_t rows) { - int32_t* index = calloc(rows, sizeof(int32_t)); + int32_t* index = taosMemoryCalloc(rows, sizeof(int32_t)); if (index == NULL) { return NULL; } @@ -878,7 +878,7 @@ static int32_t* createTupleIndex(size_t rows) { return index; } -static void destroyTupleIndex(int32_t* index) { tfree(index); } +static void destroyTupleIndex(int32_t* index) { taosMemoryFreeClear(index); } static __compar_fn_t getComparFn(int32_t type, int32_t order) { switch (type) { @@ -1019,8 +1019,8 @@ SHelper* createTupleIndex_rv(int32_t numOfRows, SArray* pOrderInfo, SSDataBlock* size_t len = sortValLengthPerRow * pBlock->info.rows; - char* buf = calloc(1, len); - SHelper* phelper = calloc(numOfRows, sizeof(SHelper)); + char* buf = taosMemoryCalloc(1, len); + SHelper* phelper = taosMemoryCalloc(numOfRows, sizeof(SHelper)); for (int32_t i = 0; i < numOfRows; ++i) { phelper[i].index = i; phelper[i].pData = buf + sortValLengthPerRow * i; @@ -1163,7 +1163,7 @@ int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRo } if (IS_VAR_DATA_TYPE(pColumn->info.type)) { - char* tmp = realloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows); + char* tmp = taosMemoryRealloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1173,9 +1173,9 @@ int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRo pColumn->varmeta.length = 0; pColumn->varmeta.allocLen = 0; - tfree(pColumn->pData); + taosMemoryFreeClear(pColumn->pData); } else { - char* tmp = realloc(pColumn->nullbitmap, BitmapLen(numOfRows)); + char* tmp = taosMemoryRealloc(pColumn->nullbitmap, BitmapLen(numOfRows)); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1183,7 +1183,7 @@ int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRo pColumn->nullbitmap = tmp; memset(pColumn->nullbitmap, 0, BitmapLen(numOfRows)); assert(pColumn->info.bytes); - tmp = realloc(pColumn->pData, numOfRows * pColumn->info.bytes); + tmp = taosMemoryRealloc(pColumn->pData, numOfRows * pColumn->info.bytes); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1214,14 +1214,14 @@ void* blockDataDestroy(SSDataBlock* pBlock) { } blockDestroyInner(pBlock); - tfree(pBlock); + taosMemoryFreeClear(pBlock); return NULL; } SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock) { int32_t numOfCols = pDataBlock->info.numOfCols; - SSDataBlock* pBlock = calloc(1, sizeof(SSDataBlock)); + SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); pBlock->info.numOfCols = numOfCols; @@ -1318,12 +1318,97 @@ int32_t tEncodeDataBlocks(void** buf, const SArray* blocks) { return tlen; } -void* tDecodeDataBlocks(const void* buf, SArray* blocks) { +void* tDecodeDataBlocks(const void* buf, SArray** blocks) { int32_t sz; buf = taosDecodeFixedI32(buf, &sz); + + *blocks = taosArrayInit(sz, sizeof(SSDataBlock)); for (int32_t i = 0; i < sz; i++) { SSDataBlock pBlock = {0}; buf = tDecodeDataBlock(buf, &pBlock); + taosArrayPush(*blocks, &pBlock); } return (void*)buf; } + +static char* formatTimestamp(char* buf, int64_t val, int precision) { + time_t tt; + int32_t ms = 0; + if (precision == TSDB_TIME_PRECISION_NANO) { + tt = (time_t)(val / 1000000000); + ms = val % 1000000000; + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + tt = (time_t)(val / 1000000); + ms = val % 1000000; + } else { + tt = (time_t)(val / 1000); + ms = val % 1000; + } + + /* comment out as it make testcases like select_with_tags.sim fail. + but in windows, this may cause the call to localtime crash if tt < 0, + need to find a better solution. + if (tt < 0) { + tt = 0; + } + */ + +#ifdef WINDOWS + if (tt < 0) tt = 0; +#endif + if (tt <= 0 && ms < 0) { + tt--; + if (precision == TSDB_TIME_PRECISION_NANO) { + ms += 1000000000; + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + ms += 1000000; + } else { + ms += 1000; + } + } + + struct tm* ptm = localtime(&tt); + size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); + + if (precision == TSDB_TIME_PRECISION_NANO) { + sprintf(buf + pos, ".%09d", ms); + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + sprintf(buf + pos, ".%06d", ms); + } else { + sprintf(buf + pos, ".%03d", ms); + } + + return buf; +} +void blockDebugShowData(SArray* dataBlocks) { + char pBuf[128]; + int32_t sz = taosArrayGetSize(dataBlocks); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(dataBlocks, i); + int32_t colNum = pDataBlock->info.numOfCols; + int32_t rows = pDataBlock->info.rows; + for (int32_t j = 0; j < rows; j++) { + printf("|"); + for (int32_t k = 0; k < colNum; k++) { + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI); + printf(" %25s |", pBuf); + break; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + printf(" %15d |", *(int32_t*)var); + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + printf(" %15ld |", *(int64_t*)var); + break; + } + } + printf("\n"); + } + } +} + diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 87e3cf0c2419b6310a1e9daf2362f3d1fff5449b..5b27fd01f4718206fa720e9fdfcef47d4ce02420 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -37,7 +37,7 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { #endif if (pCol->spaceSize < spaceNeeded) { - void *ptr = realloc(pCol->pData, spaceNeeded); + void *ptr = taosMemoryRealloc(pCol->pData, spaceNeeded); if (ptr == NULL) { uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)spaceNeeded, strerror(errno)); return -1; @@ -66,7 +66,7 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { */ STSchema *tdDupSchema(const STSchema *pSchema) { int tlen = sizeof(STSchema) + sizeof(STColumn) * schemaNCols(pSchema); - STSchema *tSchema = (STSchema *)malloc(tlen); + STSchema *tSchema = (STSchema *)taosMemoryMalloc(tlen); if (tSchema == NULL) return NULL; memcpy((void *)tSchema, (void *)pSchema, tlen); @@ -127,7 +127,7 @@ int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) { if (pBuilder == NULL) return -1; pBuilder->tCols = 256; - pBuilder->columns = (STColumn *)malloc(sizeof(STColumn) * pBuilder->tCols); + pBuilder->columns = (STColumn *)taosMemoryMalloc(sizeof(STColumn) * pBuilder->tCols); if (pBuilder->columns == NULL) return -1; tdResetTSchemaBuilder(pBuilder, version); @@ -136,7 +136,7 @@ int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) { void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder) { if (pBuilder) { - tfree(pBuilder->columns); + taosMemoryFreeClear(pBuilder->columns); } } @@ -153,7 +153,7 @@ int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int1 if (pBuilder->nCols >= pBuilder->tCols) { pBuilder->tCols *= 2; - STColumn *columns = (STColumn *)realloc(pBuilder->columns, sizeof(STColumn) * pBuilder->tCols); + STColumn *columns = (STColumn *)taosMemoryRealloc(pBuilder->columns, sizeof(STColumn) * pBuilder->tCols); if (columns == NULL) return -1; pBuilder->columns = columns; } @@ -191,7 +191,7 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) { int tlen = sizeof(STSchema) + sizeof(STColumn) * pBuilder->nCols; - STSchema *pSchema = (STSchema *)malloc(tlen); + STSchema *pSchema = (STSchema *)taosMemoryMalloc(tlen); if (pSchema == NULL) return NULL; schemaVersion(pSchema) = pBuilder->version; @@ -221,7 +221,7 @@ void tdInitDataRow(SDataRow row, STSchema *pSchema) { SDataRow tdNewDataRowFromSchema(STSchema *pSchema) { int32_t size = dataRowMaxBytesFromSchema(pSchema); - SDataRow row = malloc(size); + SDataRow row = taosMemoryMalloc(size); if (row == NULL) return NULL; tdInitDataRow(row, pSchema); @@ -232,11 +232,11 @@ SDataRow tdNewDataRowFromSchema(STSchema *pSchema) { * Free the SDataRow object */ void tdFreeDataRow(SDataRow row) { - if (row) free(row); + if (row) taosMemoryFree(row); } SDataRow tdDataRowDup(SDataRow row) { - SDataRow trow = malloc(dataRowLen(row)); + SDataRow trow = taosMemoryMalloc(dataRowLen(row)); if (trow == NULL) return NULL; dataRowCpy(trow, row); @@ -244,7 +244,7 @@ SDataRow tdDataRowDup(SDataRow row) { } SMemRow tdMemRowDup(SMemRow row) { - SMemRow trow = malloc(memRowTLen(row)); + SMemRow trow = taosMemoryMalloc(memRowTLen(row)); if (trow == NULL) return NULL; memRowCpy(trow, row); @@ -348,7 +348,7 @@ void *dataColSetOffset(SDataCol *pCol, int nEle) { } SDataCols *tdNewDataCols(int maxCols, int maxRows) { - SDataCols *pCols = (SDataCols *)calloc(1, sizeof(SDataCols)); + SDataCols *pCols = (SDataCols *)taosMemoryCalloc(1, sizeof(SDataCols)); if (pCols == NULL) { uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCols), strerror(errno)); return NULL; @@ -360,7 +360,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) { pCols->numOfCols = 0; if (maxCols > 0) { - pCols->cols = (SDataCol *)calloc(maxCols, sizeof(SDataCol)); + pCols->cols = (SDataCol *)taosMemoryCalloc(maxCols, sizeof(SDataCol)); if (pCols->cols == NULL) { uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols, strerror(errno)); @@ -384,7 +384,7 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { int oldMaxCols = pCols->maxCols; if (schemaNCols(pSchema) > oldMaxCols) { pCols->maxCols = schemaNCols(pSchema); - void *ptr = (SDataCol *)realloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols); + void *ptr = (SDataCol *)taosMemoryRealloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols); if (ptr == NULL) return -1; pCols->cols = ptr; for (i = oldMaxCols; i < pCols->maxCols; i++) { @@ -411,16 +411,17 @@ SDataCols *tdFreeDataCols(SDataCols *pCols) { int maxCols = pCols->maxCols; for (i = 0; i < maxCols; i++) { SDataCol *pCol = &pCols->cols[i]; - tfree(pCol->pData); + taosMemoryFreeClear(pCol->pData); } - free(pCols->cols); + taosMemoryFree(pCols->cols); pCols->cols = NULL; } - free(pCols); + taosMemoryFree(pCols); } return NULL; } +#if 0 SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { SDataCols *pRet = tdNewDataCols(pDataCols->maxCols, pDataCols->maxPoints); if (pRet == NULL) return NULL; @@ -454,6 +455,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { return pRet; } +#endif void tdResetDataCols(SDataCols *pCols) { if (pCols != NULL) { @@ -639,7 +641,7 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i #endif SKVRow tdKVRowDup(SKVRow row) { - SKVRow trow = malloc(kvRowLen(row)); + SKVRow trow = taosMemoryMalloc(kvRowLen(row)); if (trow == NULL) return NULL; kvRowCpy(trow, row); @@ -672,7 +674,7 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { int oRowCols = kvRowNCols(row); ASSERT(diff > 0); - nrow = malloc(nRowLen); + nrow = taosMemoryMalloc(nRowLen); if (nrow == NULL) return -1; kvRowSetLen(nrow, nRowLen); @@ -690,7 +692,7 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { tdSortKVRowByColIdx(nrow); *orow = nrow; - free(row); + taosMemoryFree(row); } else { ASSERT(((SColIdx *)ptr)->colId == colId); if (IS_VAR_DATA_TYPE(type)) { @@ -701,7 +703,7 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { } else { // need to reallocate the memory int16_t nlen = kvRowLen(row) + (varDataTLen(value) - varDataTLen(pOldVal)); ASSERT(nlen > 0); - nrow = malloc(nlen); + nrow = taosMemoryMalloc(nlen); if (nrow == NULL) return -1; kvRowSetLen(nrow, nlen); @@ -726,7 +728,7 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { } *orow = nrow; - free(row); + taosMemoryFree(row); } } else { memcpy(kvRowColVal(row, (SColIdx *)ptr), value, TYPE_BYTES[type]); @@ -755,21 +757,21 @@ void *tdDecodeKVRow(void *buf, SKVRow *row) { int tdInitKVRowBuilder(SKVRowBuilder *pBuilder) { pBuilder->tCols = 128; pBuilder->nCols = 0; - pBuilder->pColIdx = (SColIdx *)malloc(sizeof(SColIdx) * pBuilder->tCols); + pBuilder->pColIdx = (SColIdx *)taosMemoryMalloc(sizeof(SColIdx) * pBuilder->tCols); if (pBuilder->pColIdx == NULL) return -1; pBuilder->alloc = 1024; pBuilder->size = 0; - pBuilder->buf = malloc(pBuilder->alloc); + pBuilder->buf = taosMemoryMalloc(pBuilder->alloc); if (pBuilder->buf == NULL) { - free(pBuilder->pColIdx); + taosMemoryFree(pBuilder->pColIdx); return -1; } return 0; } void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder) { - tfree(pBuilder->pColIdx); - tfree(pBuilder->buf); + taosMemoryFreeClear(pBuilder->pColIdx); + taosMemoryFreeClear(pBuilder->buf); } void tdResetKVRowBuilder(SKVRowBuilder *pBuilder) { @@ -783,7 +785,7 @@ SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) { tlen += TD_KV_ROW_HEAD_SIZE; - SKVRow row = malloc(tlen); + SKVRow row = taosMemoryMalloc(tlen); if (row == NULL) return NULL; kvRowSetNCols(row, pBuilder->nCols); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 841824a7c75c4d5c9f2fb858e7690f29d266ca6b..179a50c4226a635811fe9d6b27755138c9810f0e 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -132,6 +132,9 @@ bool tsdbForceKeepFile = false; int32_t tsDiskCfgNum = 0; SDiskCfg tsDiskCfg[TFS_MAX_DISKS] = {0}; +// stream scheduler +bool tsStreamSchedV = true; + /* * minimum scale for whole system, millisecond by default * for TSDB_TIME_PRECISION_MILLI: 86400000L @@ -177,6 +180,10 @@ static int32_t taosSetTfsCfg(SConfig *pCfg) { memcpy(&tsDiskCfg[index], pCfg, sizeof(SDiskCfg)); if (pCfg->level == 0 && pCfg->primary == 1) { tstrncpy(tsDataDir, pCfg->dir, PATH_MAX); + if (taosMkDir(tsDataDir) != 0) { + uError("failed to create dataDir:%s since %s", tsDataDir, terrstr()); + return -1; + } } if (taosMkDir(pCfg->dir) != 0) { uError("failed to create tfsDir:%s since %s", tsDataDir, terrstr()); @@ -585,4 +592,4 @@ void taosCfgDynamicOptions(const char *option, const char *value) { taosResetLog(); cfgDumpCfg(tsCfg, 1, false); } -} \ No newline at end of file +} diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 1ea00566f483e87340d8ff2a9816af1b4901e2fb..d7df3e115f9a7430750f2040d4b60c0f638dfadd 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -28,7 +28,7 @@ #undef TD_MSG_SEG_CODE_ #include "tmsgdef.h" -int32_t tInitSubmitMsgIter(SSubmitReq *pMsg, SSubmitMsgIter *pIter) { +int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { if (pMsg == NULL) { terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; return -1; @@ -287,6 +287,7 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pReq->ver); + tlen += taosEncodeString(buf, pReq->dbFName); tlen += taosEncodeString(buf, pReq->name); tlen += taosEncodeFixedU32(buf, pReq->ttl); tlen += taosEncodeFixedU32(buf, pReq->keep); @@ -313,12 +314,12 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) { tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pBSmaCols[i]); } - if(pReq->rollup && NULL != pReq->stbCfg.pRSmaParam) { + if (pReq->rollup && pReq->stbCfg.pRSmaParam) { SRSmaParam *param = pReq->stbCfg.pRSmaParam; tlen += taosEncodeFixedU32(buf, (uint32_t)param->xFilesFactor); tlen += taosEncodeFixedI8(buf, param->delayUnit); tlen += taosEncodeFixedI8(buf, param->nFuncIds); - for(int8_t i=0; i< param->nFuncIds; ++i) { + for (int8_t i = 0; i < param->nFuncIds; ++i) { tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]); } tlen += taosEncodeFixedI64(buf, param->delay); @@ -336,16 +337,16 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].bytes); tlen += taosEncodeString(buf, pReq->ntbCfg.pSchema[i].name); } - tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nBSmaCols); - for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) { - tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pBSmaCols[i]); + tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nBSmaCols); + for (col_id_t i = 0; i < pReq->ntbCfg.nBSmaCols; ++i) { + tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.pBSmaCols[i]); } - if(pReq->rollup && NULL != pReq->stbCfg.pRSmaParam) { - SRSmaParam *param = pReq->stbCfg.pRSmaParam; + if (pReq->rollup && pReq->ntbCfg.pRSmaParam) { + SRSmaParam *param = pReq->ntbCfg.pRSmaParam; tlen += taosEncodeFixedU32(buf, (uint32_t)param->xFilesFactor); tlen += taosEncodeFixedI8(buf, param->delayUnit); tlen += taosEncodeFixedI8(buf, param->nFuncIds); - for(int8_t i=0; i< param->nFuncIds; ++i) { + for (int8_t i = 0; i < param->nFuncIds; ++i) { tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]); } tlen += taosEncodeFixedI64(buf, param->delay); @@ -360,6 +361,7 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { buf = taosDecodeFixedI64(buf, &(pReq->ver)); + buf = taosDecodeString(buf, &(pReq->dbFName)); buf = taosDecodeString(buf, &(pReq->name)); buf = taosDecodeFixedU32(buf, &(pReq->ttl)); buf = taosDecodeFixedU32(buf, &(pReq->keep)); @@ -369,7 +371,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { case TD_SUPER_TABLE: buf = taosDecodeFixedI64(buf, &(pReq->stbCfg.suid)); buf = taosDecodeFixedU32(buf, &(pReq->stbCfg.nCols)); - pReq->stbCfg.pSchema = (SSchema *)malloc(pReq->stbCfg.nCols * sizeof(SSchema)); + pReq->stbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchema)); for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) { buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].type)); buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].colId)); @@ -377,7 +379,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { buf = taosDecodeStringTo(buf, pReq->stbCfg.pSchema[i].name); } buf = taosDecodeFixedU32(buf, &pReq->stbCfg.nTagCols); - pReq->stbCfg.pTagSchema = (SSchema *)malloc(pReq->stbCfg.nTagCols * sizeof(SSchema)); + pReq->stbCfg.pTagSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nTagCols * sizeof(SSchema)); for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) { buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type)); buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].colId); @@ -385,22 +387,22 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name); } buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols)); - if(pReq->stbCfg.nBSmaCols > 0) { - pReq->stbCfg.pBSmaCols = (col_id_t *)malloc(pReq->stbCfg.nBSmaCols * sizeof(col_id_t)); + if (pReq->stbCfg.nBSmaCols > 0) { + pReq->stbCfg.pBSmaCols = (col_id_t *)taosMemoryMalloc(pReq->stbCfg.nBSmaCols * sizeof(col_id_t)); for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) { buf = taosDecodeFixedI16(buf, pReq->stbCfg.pBSmaCols + i); } } else { pReq->stbCfg.pBSmaCols = NULL; } - if(pReq->rollup) { - pReq->stbCfg.pRSmaParam = (SRSmaParam *)malloc(sizeof(SRSmaParam)); + if (pReq->rollup) { + pReq->stbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam)); SRSmaParam *param = pReq->stbCfg.pRSmaParam; - buf = taosDecodeFixedU32(buf, (uint32_t*)¶m->xFilesFactor); + buf = taosDecodeFixedU32(buf, (uint32_t *)¶m->xFilesFactor); buf = taosDecodeFixedI8(buf, ¶m->delayUnit); buf = taosDecodeFixedI8(buf, ¶m->nFuncIds); - if(param->nFuncIds > 0) { - for (int8_t i = 0; i< param->nFuncIds; ++i) { + if (param->nFuncIds > 0) { + for (int8_t i = 0; i < param->nFuncIds; ++i) { buf = taosDecodeFixedI32(buf, param->pFuncIds + i); } } else { @@ -417,30 +419,30 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { break; case TD_NORMAL_TABLE: buf = taosDecodeFixedU32(buf, &pReq->ntbCfg.nCols); - pReq->ntbCfg.pSchema = (SSchema *)malloc(pReq->ntbCfg.nCols * sizeof(SSchema)); + pReq->ntbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchema)); for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) { buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].type); buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].colId); buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].bytes); buf = taosDecodeStringTo(buf, pReq->ntbCfg.pSchema[i].name); } - buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols)); - if(pReq->stbCfg.nBSmaCols > 0) { - pReq->stbCfg.pBSmaCols = (col_id_t *)malloc(pReq->stbCfg.nBSmaCols * sizeof(col_id_t)); - for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) { - buf = taosDecodeFixedI16(buf, pReq->stbCfg.pBSmaCols + i); + buf = taosDecodeFixedI16(buf, &(pReq->ntbCfg.nBSmaCols)); + if (pReq->ntbCfg.nBSmaCols > 0) { + pReq->ntbCfg.pBSmaCols = (col_id_t *)taosMemoryMalloc(pReq->ntbCfg.nBSmaCols * sizeof(col_id_t)); + for (col_id_t i = 0; i < pReq->ntbCfg.nBSmaCols; ++i) { + buf = taosDecodeFixedI16(buf, pReq->ntbCfg.pBSmaCols + i); } } else { - pReq->stbCfg.pBSmaCols = NULL; + pReq->ntbCfg.pBSmaCols = NULL; } - if(pReq->rollup) { - pReq->stbCfg.pRSmaParam = (SRSmaParam *)malloc(sizeof(SRSmaParam)); - SRSmaParam *param = pReq->stbCfg.pRSmaParam; - buf = taosDecodeFixedU32(buf, (uint32_t*)¶m->xFilesFactor); + if (pReq->rollup) { + pReq->ntbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam)); + SRSmaParam *param = pReq->ntbCfg.pRSmaParam; + buf = taosDecodeFixedU32(buf, (uint32_t *)¶m->xFilesFactor); buf = taosDecodeFixedI8(buf, ¶m->delayUnit); buf = taosDecodeFixedI8(buf, ¶m->nFuncIds); - if(param->nFuncIds > 0) { - for (int8_t i = 0; i< param->nFuncIds; ++i) { + if (param->nFuncIds > 0) { + for (int8_t i = 0; i < param->nFuncIds; ++i) { buf = taosDecodeFixedI32(buf, param->pFuncIds + i); } } else { @@ -448,7 +450,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { } buf = taosDecodeFixedI64(buf, ¶m->delay); } else { - pReq->stbCfg.pRSmaParam = NULL; + pReq->ntbCfg.pRSmaParam = NULL; } break; default: @@ -478,7 +480,7 @@ void *tDeserializeSVCreateTbBatchReq(void *buf, SVCreateTbBatchReq *pReq) { buf = taosDecodeFixedU32(buf, &nsize); pReq->pArray = taosArrayInit(nsize, sizeof(SVCreateTbReq)); for (size_t i = 0; i < nsize; i++) { - SVCreateTbReq req; + SVCreateTbReq req = {0}; buf = tDeserializeSVCreateTbReq(buf, &req); taosArrayPush(pReq->pArray, &req); } @@ -508,8 +510,14 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq if (tStartEncode(&encoder) < 0) return -1; if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; + if (tEncodeFloat(&encoder, pReq->xFilesFactor) < 0) return -1; + if (tEncodeI32(&encoder, pReq->aggregationMethod) < 0) return -1; + if (tEncodeI32(&encoder, pReq->delay) < 0) return -1; + if (tEncodeI32(&encoder, pReq->ttl) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfColumns) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfTags) < 0) return -1; + if (tEncodeI32(&encoder, pReq->numOfSmas) < 0) return -1; + if (tEncodeI32(&encoder, pReq->commentLen) < 0) return -1; for (int32_t i = 0; i < pReq->numOfColumns; ++i) { SField *pField = taosArrayGet(pReq->pColumns, i); @@ -525,7 +533,16 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq if (tEncodeCStr(&encoder, pField->name) < 0) return -1; } - if (tEncodeCStr(&encoder, pReq->comment) < 0) return -1; + for (int32_t i = 0; i < pReq->numOfSmas; ++i) { + SField *pField = taosArrayGet(pReq->pSmas, i); + if (tEncodeI8(&encoder, pField->type) < 0) return -1; + if (tEncodeI32(&encoder, pField->bytes) < 0) return -1; + if (tEncodeCStr(&encoder, pField->name) < 0) return -1; + } + + if (pReq->commentLen > 0) { + if (tEncodeBinary(&encoder, pReq->comment, pReq->commentLen) < 0) return -1; + } tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -540,12 +557,19 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR if (tStartDecode(&decoder) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1; + if (tDecodeFloat(&decoder, &pReq->xFilesFactor) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->aggregationMethod) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->delay) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->ttl) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfColumns) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfTags) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->numOfSmas) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->commentLen) < 0) return -1; pReq->pColumns = taosArrayInit(pReq->numOfColumns, sizeof(SField)); pReq->pTags = taosArrayInit(pReq->numOfTags, sizeof(SField)); - if (pReq->pColumns == NULL || pReq->pTags == NULL) { + pReq->pSmas = taosArrayInit(pReq->numOfSmas, sizeof(SField)); + if (pReq->pColumns == NULL || pReq->pTags == NULL || pReq->pSmas == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -572,7 +596,23 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR } } - if (tDecodeCStrTo(&decoder, pReq->comment) < 0) return -1; + for (int32_t i = 0; i < pReq->numOfSmas; ++i) { + SField field = {0}; + if (tDecodeI8(&decoder, &field.type) < 0) return -1; + if (tDecodeI32(&decoder, &field.bytes) < 0) return -1; + if (tDecodeCStrTo(&decoder, field.name) < 0) return -1; + if (taosArrayPush(pReq->pSmas, &field) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + } + + if (pReq->commentLen > 0) { + pReq->comment = taosMemoryMalloc(pReq->commentLen); + if (pReq->comment == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->comment) < 0) return -1; + } + tEndDecode(&decoder); tCoderClear(&decoder); @@ -582,8 +622,11 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR void tFreeSMCreateStbReq(SMCreateStbReq *pReq) { taosArrayDestroy(pReq->pColumns); taosArrayDestroy(pReq->pTags); + taosArrayDestroy(pReq->pSmas); + taosMemoryFreeClear(pReq->comment); pReq->pColumns = NULL; pReq->pTags = NULL; + pReq->pSmas = NULL; } int32_t tSerializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq) { @@ -669,6 +712,123 @@ void tFreeSMAltertbReq(SMAltertbReq *pReq) { pReq->pFields = NULL; } +int32_t tSerializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->stb) < 0) return -1; + if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; + if (tEncodeI8(&encoder, pReq->intervalUnit) < 0) return -1; + if (tEncodeI8(&encoder, pReq->slidingUnit) < 0) return -1; + if (tEncodeI8(&encoder, pReq->timezone) < 0) return -1; + if (tEncodeI32(&encoder, pReq->dstVgId) < 0) return -1; + if (tEncodeI64(&encoder, pReq->interval) < 0) return -1; + if (tEncodeI64(&encoder, pReq->offset) < 0) return -1; + if (tEncodeI64(&encoder, pReq->sliding) < 0) return -1; + if (tEncodeI32(&encoder, pReq->exprLen) < 0) return -1; + if (tEncodeI32(&encoder, pReq->tagsFilterLen) < 0) return -1; + if (tEncodeI32(&encoder, pReq->sqlLen) < 0) return -1; + if (tEncodeI32(&encoder, pReq->astLen) < 0) return -1; + if (pReq->exprLen > 0) { + if (tEncodeBinary(&encoder, pReq->expr, pReq->exprLen) < 0) return -1; + } + if (pReq->tagsFilterLen > 0) { + if (tEncodeBinary(&encoder, pReq->tagsFilter, pReq->tagsFilterLen) < 0) return -1; + } + if (pReq->sqlLen > 0) { + if (tEncodeBinary(&encoder, pReq->sql, pReq->sqlLen) < 0) return -1; + } + if (pReq->astLen > 0) { + if (tEncodeBinary(&encoder, pReq->ast, pReq->astLen) < 0) return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->stb) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->intervalUnit) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->slidingUnit) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->timezone) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->dstVgId) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->interval) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->offset) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->sliding) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->exprLen) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->tagsFilterLen) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->sqlLen) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->astLen) < 0) return -1; + if (pReq->exprLen > 0) { + pReq->expr = taosMemoryMalloc(pReq->exprLen); + if (pReq->expr == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->expr) < 0) return -1; + } + if (pReq->tagsFilterLen > 0) { + pReq->tagsFilter = taosMemoryMalloc(pReq->tagsFilterLen); + if (pReq->tagsFilter == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->tagsFilter) < 0) return -1; + } + if (pReq->sqlLen > 0) { + pReq->sql = taosMemoryMalloc(pReq->sqlLen); + if (pReq->sql == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->sql) < 0) return -1; + } + if (pReq->astLen > 0) { + pReq->ast = taosMemoryMalloc(pReq->astLen); + if (pReq->ast == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1; + } + + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; +} + +void tFreeSMCreateSmaReq(SMCreateSmaReq *pReq) { + taosMemoryFreeClear(pReq->expr); + taosMemoryFreeClear(pReq->tagsFilter); + taosMemoryFreeClear(pReq->sql); + taosMemoryFreeClear(pReq->ast); +} + +int32_t tSerializeSMDropSmaReq(void *buf, int32_t bufLen, SMDropSmaReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; + if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSMDropSmaReq(void *buf, int32_t bufLen, SMDropSmaReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -1384,6 +1544,14 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) { if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1; if (tEncodeI8(&encoder, pReq->ignoreExist) < 0) return -1; if (tEncodeI8(&encoder, pReq->streamMode) < 0) return -1; + if (tEncodeI32(&encoder, pReq->numOfRetensions) < 0) return -1; + for (int32_t i = 0; i < pReq->numOfRetensions; ++i) { + SRetention *pRetension = taosArrayGet(pReq->pRetensions, i); + if (tEncodeI32(&encoder, pRetension->freq) < 0) return -1; + if (tEncodeI32(&encoder, pRetension->keep) < 0) return -1; + if (tEncodeI8(&encoder, pRetension->freqUnit) < 0) return -1; + if (tEncodeI8(&encoder, pRetension->keepUnit) < 0) return -1; + } tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1417,12 +1585,36 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1; if (tDecodeI8(&decoder, &pReq->ignoreExist) < 0) return -1; if (tDecodeI8(&decoder, &pReq->streamMode) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->numOfRetensions) < 0) return -1; + pReq->pRetensions = taosArrayInit(pReq->numOfRetensions, sizeof(SRetention)); + if (pReq->pRetensions == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < pReq->numOfRetensions; ++i) { + SRetention rentension = {0}; + if (tDecodeI32(&decoder, &rentension.freq) < 0) return -1; + if (tDecodeI32(&decoder, &rentension.keep) < 0) return -1; + if (tDecodeI8(&decoder, &rentension.freqUnit) < 0) return -1; + if (tDecodeI8(&decoder, &rentension.keepUnit) < 0) return -1; + if (taosArrayPush(pReq->pRetensions, &rentension) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + } + tEndDecode(&decoder); tCoderClear(&decoder); return 0; } +void tFreeSCreateDbReq(SCreateDbReq *pReq) { + taosArrayDestroy(pReq->pRetensions); + pReq->pRetensions = NULL; +} + int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -1795,7 +1987,7 @@ int32_t tDeserializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) { if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; if (tDecodeI32(&decoder, &pReq->payloadLen) < 0) return -1; if (pReq->payloadLen > 0) { - pReq->payload = malloc(pReq->payloadLen); + pReq->payload = taosMemoryMalloc(pReq->payloadLen); if (pReq->payload == NULL) return -1; if (tDecodeCStrTo(&decoder, pReq->payload) < 0) return -1; } @@ -1805,7 +1997,7 @@ int32_t tDeserializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) { return 0; } -void tFreeSShowReq(SShowReq *pReq) { tfree(pReq->payload); } +void tFreeSShowReq(SShowReq *pReq) { taosMemoryFreeClear(pReq->payload); } int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq *pReq) { SCoder encoder = {0}; @@ -1879,7 +2071,7 @@ static int32_t tDecodeSTableMetaRsp(SCoder *pDecoder, STableMetaRsp *pRsp) { if (tDecodeI32(pDecoder, &pRsp->vgId) < 0) return -1; int32_t totalCols = pRsp->numOfTags + pRsp->numOfColumns; - pRsp->pSchemas = malloc(sizeof(SSchema) * totalCols); + pRsp->pSchemas = taosMemoryMalloc(sizeof(SSchema) * totalCols); if (pRsp->pSchemas == NULL) return -1; for (int32_t i = 0; i < totalCols; ++i) { @@ -1960,7 +2152,7 @@ int32_t tDeserializeSTableMetaBatchRsp(void *buf, int32_t bufLen, STableMetaBatc return 0; } -void tFreeSTableMetaRsp(STableMetaRsp *pRsp) { tfree(pRsp->pSchemas); } +void tFreeSTableMetaRsp(STableMetaRsp *pRsp) { taosMemoryFreeClear(pRsp->pSchemas); } void tFreeSTableMetaBatchRsp(STableMetaBatchRsp *pRsp) { int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); @@ -2112,13 +2304,13 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR if (tDecodeI32(&decoder, &astLen) < 0) return -1; if (sqlLen > 0) { - pReq->sql = calloc(1, sqlLen + 1); + pReq->sql = taosMemoryCalloc(1, sqlLen + 1); if (pReq->sql == NULL) return -1; if (tDecodeCStrTo(&decoder, pReq->sql) < 0) return -1; } if (astLen > 0) { - pReq->ast = calloc(1, astLen + 1); + pReq->ast = taosMemoryCalloc(1, astLen + 1); if (pReq->ast == NULL) return -1; if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1; } @@ -2130,8 +2322,8 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR } void tFreeSCMCreateTopicReq(SCMCreateTopicReq *pReq) { - tfree(pReq->sql); - tfree(pReq->ast); + taosMemoryFreeClear(pReq->sql); + taosMemoryFreeClear(pReq->ast); } int32_t tSerializeSCMCreateTopicRsp(void *buf, int32_t bufLen, const SCMCreateTopicRsp *pRsp) { @@ -2300,6 +2492,14 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR SReplica *pReplica = &pReq->replicas[i]; if (tEncodeSReplica(&encoder, pReplica) < 0) return -1; } + if (tEncodeI32(&encoder, pReq->numOfRetensions) < 0) return -1; + for (int32_t i = 0; i < pReq->numOfRetensions; ++i) { + SRetention *pRetension = taosArrayGet(pReq->pRetensions, i); + if (tEncodeI32(&encoder, pRetension->freq) < 0) return -1; + if (tEncodeI32(&encoder, pRetension->keep) < 0) return -1; + if (tEncodeI8(&encoder, pRetension->freqUnit) < 0) return -1; + if (tEncodeI8(&encoder, pRetension->keepUnit) < 0) return -1; + } tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -2344,11 +2544,35 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; } + if (tDecodeI32(&decoder, &pReq->numOfRetensions) < 0) return -1; + pReq->pRetensions = taosArrayInit(pReq->numOfRetensions, sizeof(SRetention)); + if (pReq->pRetensions == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < pReq->numOfRetensions; ++i) { + SRetention rentension = {0}; + if (tDecodeI32(&decoder, &rentension.freq) < 0) return -1; + if (tDecodeI32(&decoder, &rentension.keep) < 0) return -1; + if (tDecodeI8(&decoder, &rentension.freqUnit) < 0) return -1; + if (tDecodeI8(&decoder, &rentension.keepUnit) < 0) return -1; + if (taosArrayPush(pReq->pRetensions, &rentension) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + } + tEndDecode(&decoder); tCoderClear(&decoder); return 0; } +int32_t tFreeSCreateVnodeReq(SCreateVnodeReq *pReq) { + taosArrayDestroy(pReq->pRetensions); + pReq->pRetensions = NULL; +} + int32_t tSerializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -2709,7 +2933,7 @@ int32_t tSerializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pRsp tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeI32(&encoder, pRsp->code) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->code) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -2736,13 +2960,13 @@ int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchR if (tStartEncode(&encoder) < 0) return -1; if (pRsp->rspList) { int32_t num = taosArrayGetSize(pRsp->rspList); - if (tEncodeI32(&encoder, num) < 0) return -1; + if (tEncodeI32(&encoder, num) < 0) return -1; for (int32_t i = 0; i < num; ++i) { SVCreateTbRsp *rsp = taosArrayGet(pRsp->rspList, i); - if (tEncodeI32(&encoder, rsp->code) < 0) return -1; + if (tEncodeI32(&encoder, rsp->code) < 0) return -1; } } else { - if (tEncodeI32(&encoder, 0) < 0) return -1; + if (tEncodeI32(&encoder, 0) < 0) return -1; } tEndEncode(&encoder); @@ -2752,7 +2976,7 @@ int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchR } int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) { - SCoder decoder = {0}; + SCoder decoder = {0}; int32_t num = 0; tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); @@ -2775,7 +2999,6 @@ int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatc return 0; } - int32_t tSerializeSVCreateTSmaReq(void **buf, SVCreateTSmaReq *pReq) { int32_t tlen = 0; @@ -2822,9 +3045,10 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS if (tStartEncode(&encoder) < 0) return -1; if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->outputTbName) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->outputSTbName) < 0) return -1; if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1; + if (tEncodeI32(&encoder, sqlLen) < 0) return -1; + if (tEncodeI32(&encoder, astLen) < 0) return -1; if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1; if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1; @@ -2844,19 +3068,19 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea if (tStartDecode(&decoder) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->outputTbName) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->outputSTbName) < 0) return -1; if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1; if (tDecodeI32(&decoder, &sqlLen) < 0) return -1; if (tDecodeI32(&decoder, &astLen) < 0) return -1; if (sqlLen > 0) { - pReq->sql = calloc(1, sqlLen + 1); + pReq->sql = taosMemoryCalloc(1, sqlLen + 1); if (pReq->sql == NULL) return -1; if (tDecodeCStrTo(&decoder, pReq->sql) < 0) return -1; } if (astLen > 0) { - pReq->ast = calloc(1, astLen + 1); + pReq->ast = taosMemoryCalloc(1, astLen + 1); if (pReq->ast == NULL) return -1; if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1; } @@ -2867,41 +3091,6 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea } void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) { - tfree(pReq->sql); - tfree(pReq->ast); -} - -int32_t tEncodeSStreamTask(SCoder *pEncoder, const SStreamTask *pTask) { - if (tStartEncode(pEncoder) < 0) return -1; - if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1; - if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1; - if (tEncodeI32(pEncoder, pTask->level) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->status) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->pipeSink) < 0) return -1; - // if (tEncodeI8(pEncoder, pTask->numOfRunners) < 0) return -1; - if (tEncodeSEpSet(pEncoder, &pTask->NextOpEp) < 0) return -1; - if (tEncodeCStr(pEncoder, pTask->qmsg) < 0) return -1; - tEndEncode(pEncoder); - return pEncoder->pos; -} - -int32_t tDecodeSStreamTask(SCoder *pDecoder, SStreamTask *pTask) { - if (tStartDecode(pDecoder) < 0) return -1; - if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1; - if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1; - if (tDecodeI32(pDecoder, &pTask->level) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->pipeSink) < 0) return -1; - // if (tDecodeI8(pDecoder, &pTask->numOfRunners) < 0) return -1; - if (tDecodeSEpSet(pDecoder, &pTask->NextOpEp) < 0) return -1; - if (tDecodeCStrAlloc(pDecoder, &pTask->qmsg) < 0) return -1; - tEndDecode(pDecoder); - return 0; -} - -void tFreeSStreamTask(SStreamTask *pTask) { - // TODO - /*free(pTask->qmsg);*/ - /*free(pTask->executor);*/ - /*free(pTask);*/ + taosMemoryFreeClear(pReq->sql); + taosMemoryFreeClear(pReq->ast); } diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 5561eb93c3dd8db15d4eac1f89e3869915a88325..f4755f5b5e4e924ed0c917701430c97c4e08f5ef 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -30,13 +30,13 @@ SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFil return NULL; } - SColumnFilterInfo* pFilter = calloc(1, numOfFilters * sizeof(SColumnFilterInfo)); + SColumnFilterInfo* pFilter = taosMemoryCalloc(1, numOfFilters * sizeof(SColumnFilterInfo)); memcpy(pFilter, src, sizeof(SColumnFilterInfo) * numOfFilters); for (int32_t j = 0; j < numOfFilters; ++j) { if (pFilter[j].filterstr) { size_t len = (size_t) pFilter[j].len + 1 * TSDB_NCHAR_SIZE; - pFilter[j].pz = (int64_t) calloc(1, len); + pFilter[j].pz = (int64_t) taosMemoryCalloc(1, len); memcpy((char*)pFilter[j].pz, (char*)src[j].pz, (size_t) pFilter[j].len); } @@ -171,7 +171,7 @@ bool tNameIsValid(const SName* name) { SName* tNameDup(const SName* name) { assert(name != NULL); - SName* p = malloc(sizeof(SName)); + SName* p = taosMemoryMalloc(sizeof(SName)); memcpy(p, name, sizeof(SName)); return p; } diff --git a/source/common/src/trow.c b/source/common/src/trow.c index db4bc49425e1835f3451bea18049f4488a1c2b42..9ee2ec1300487b5a46e3a63a2f8b8881b639b2ab 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -78,7 +78,7 @@ static FORCE_INLINE void dataColSetNoneAt(SDataCol *pCol, int index, bool setBit setNull(POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * index), pCol->type, pCol->bytes); pCol->len += TYPE_BYTES[pCol->type]; } - if(setBitmap) { + if (setBitmap) { tdSetBitmapValType(pCol->pBitmap, index, TD_VTYPE_NONE); } } @@ -118,8 +118,8 @@ int trbWriteCol(SRowBuilder *pRB, void *pData, col_id_t cid) { #endif -STSRow* tdRowDup(STSRow *row) { - STSRow* trow = malloc(TD_ROW_LEN(row)); +STSRow *tdRowDup(STSRow *row) { + STSRow *trow = taosMemoryMalloc(TD_ROW_LEN(row)); if (trow == NULL) return NULL; tdRowCpy(trow, row); @@ -176,7 +176,7 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols SDataCol *pDataCol = &(pCols->cols[0]); if (pDataCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints); } while (dcol < pCols->numOfCols) { @@ -378,9 +378,7 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i } } - - -STSRow* mergeTwoRows(void *buffer, STSRow* row1, STSRow *row2, STSchema *pSchema1, STSchema *pSchema2) { +STSRow *mergeTwoRows(void *buffer, STSRow *row1, STSRow *row2, STSchema *pSchema1, STSchema *pSchema2) { #if 0 ASSERT(TD_ROW_KEY(row1) == TD_ROW_KEY(row2)); ASSERT(schemaVersion(pSchema1) == TD_ROW_SVER(row1)); @@ -473,6 +471,44 @@ STSRow* mergeTwoRows(void *buffer, STSRow* row1, STSRow *row2, STSchema *pSchema } taosArrayDestroy(stashRow); return buffer; - #endif +#endif return NULL; } + +SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { + SDataCols *pRet = tdNewDataCols(pDataCols->maxCols, pDataCols->maxPoints); + if (pRet == NULL) return NULL; + + pRet->numOfCols = pDataCols->numOfCols; + pRet->sversion = pDataCols->sversion; + if (keepData) pRet->numOfRows = pDataCols->numOfRows; + + for (int i = 0; i < pDataCols->numOfCols; i++) { + pRet->cols[i].type = pDataCols->cols[i].type; + pRet->cols[i].bitmap = pDataCols->cols[i].bitmap; + pRet->cols[i].colId = pDataCols->cols[i].colId; + pRet->cols[i].bytes = pDataCols->cols[i].bytes; + pRet->cols[i].offset = pDataCols->cols[i].offset; + + if (keepData) { + if (pDataCols->cols[i].len > 0) { + if (tdAllocMemForCol(&pRet->cols[i], pRet->maxPoints) < 0) { + tdFreeDataCols(pRet); + return NULL; + } + pRet->cols[i].len = pDataCols->cols[i].len; + memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len); + if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) { + int dataOffSize = sizeof(VarDataOffsetT) * pDataCols->maxPoints; + memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, dataOffSize); + } + if (!TD_COL_ROWS_NORM(pRet->cols + i)) { + int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(pDataCols->maxPoints); + memcpy(pRet->cols[i].pBitmap, pDataCols->cols[i].pBitmap, nBitmapBytes); + } + } + } + } + + return pRet; +} \ No newline at end of file diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 22c10f62b4ee11df24af06b8131b4dde89e9be27..510a2809e74b92acecc510b928d9fe288ba9f7db 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -458,16 +458,21 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { if (duration == 0) { return t; } - if (unit == 'y') { - duration *= 12; - } else if (unit != 'n') { + + if (unit != 'n' && unit != 'y') { return t + duration; } + // The following code handles the y/n time duration + int64_t numOfMonth = duration; + if (unit == 'y') { + numOfMonth *= 12; + } + struct tm tm; time_t tt = (time_t)(t / TSDB_TICK_PER_SECOND(precision)); taosLocalTime(&tt, &tm); - int32_t mon = tm.tm_year * 12 + tm.tm_mon + (int32_t)duration; + int32_t mon = tm.tm_year * 12 + tm.tm_mon + (int32_t)numOfMonth; tm.tm_year = mon / 12; tm.tm_mon = mon % 12; @@ -557,8 +562,7 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio // not enough time range if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { - end = start + pInterval->interval - 1; - + end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; while (end < t && ((start + pInterval->sliding) <= INT64_MAX)) { // move forward to the correct time window start += pInterval->sliding; @@ -574,12 +578,23 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio } } + ASSERT(pInterval->offset >= 0); + if (pInterval->offset > 0) { start = taosTimeAdd(start, pInterval->offset, pInterval->offsetUnit, precision); if (start > t) { start = taosTimeAdd(start, -pInterval->interval, pInterval->intervalUnit, precision); + } else { + // try to move current window to the left-hande-side, due to the offset effect. + int64_t end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; + ASSERT(end >= t); + end = taosTimeAdd(end, -pInterval->sliding, pInterval->slidingUnit, precision); + if (end >= t) { + start = taosTimeAdd(start, -pInterval->sliding, pInterval->slidingUnit, precision); + } } } + return start; } diff --git a/source/common/src/ttszip.c b/source/common/src/ttszip.c index 5f0a3532268f3222d3852d5b31e7f999fe307ff9..15e741d3076161f33738124dadfaad551bb56bca 100644 --- a/source/common/src/ttszip.c +++ b/source/common/src/ttszip.c @@ -30,7 +30,7 @@ static int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader); * @return */ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { - STSBuf* pTSBuf = calloc(1, sizeof(STSBuf)); + STSBuf* pTSBuf = taosMemoryCalloc(1, sizeof(STSBuf)); if (pTSBuf == NULL) { return NULL; } @@ -41,7 +41,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { // pTSBuf->pFile = fopen(pTSBuf->path, "wb+"); pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); if (pTSBuf->pFile == NULL) { - free(pTSBuf); + taosMemoryFree(pTSBuf); return NULL; } @@ -66,7 +66,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { } STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { - STSBuf* pTSBuf = calloc(1, sizeof(STSBuf)); + STSBuf* pTSBuf = taosMemoryCalloc(1, sizeof(STSBuf)); if (pTSBuf == NULL) { return NULL; } @@ -78,7 +78,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { // pTSBuf->pFile = fopen(pTSBuf->path, "rb+"); pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_WRITE | TD_FILE_READ); if (pTSBuf->pFile == NULL) { - free(pTSBuf); + taosMemoryFree(pTSBuf); return NULL; } @@ -101,7 +101,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { if (header.numOfGroup > pTSBuf->numOfAlloc) { pTSBuf->numOfAlloc = header.numOfGroup; - STSGroupBlockInfoEx* tmp = realloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * pTSBuf->numOfAlloc); + STSGroupBlockInfoEx* tmp = taosMemoryRealloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * pTSBuf->numOfAlloc); if (tmp == NULL) { tsBufDestroy(pTSBuf); return NULL; @@ -122,7 +122,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { size_t infoSize = sizeof(STSGroupBlockInfo) * pTSBuf->numOfGroups; - STSGroupBlockInfo* buf = (STSGroupBlockInfo*)calloc(1, infoSize); + STSGroupBlockInfo* buf = (STSGroupBlockInfo*)taosMemoryCalloc(1, infoSize); if (buf == NULL) { tsBufDestroy(pTSBuf); return NULL; @@ -137,7 +137,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { STSGroupBlockInfoEx* pBlockList = &pTSBuf->pData[i]; memcpy(&pBlockList->info, &buf[i], sizeof(STSGroupBlockInfo)); } - free(buf); + taosMemoryFree(buf); ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_END); UNUSED(ret); @@ -166,11 +166,11 @@ void* tsBufDestroy(STSBuf* pTSBuf) { return NULL; } - tfree(pTSBuf->assistBuf); - tfree(pTSBuf->tsData.rawBuf); + taosMemoryFreeClear(pTSBuf->assistBuf); + taosMemoryFreeClear(pTSBuf->tsData.rawBuf); - tfree(pTSBuf->pData); - tfree(pTSBuf->block.payload); + taosMemoryFreeClear(pTSBuf->pData); + taosMemoryFreeClear(pTSBuf->block.payload); if (!pTSBuf->remainOpen) { taosCloseFile(&pTSBuf->pFile); @@ -184,7 +184,7 @@ void* tsBufDestroy(STSBuf* pTSBuf) { } taosVariantDestroy(&pTSBuf->block.tag); - free(pTSBuf); + taosMemoryFree(pTSBuf); return NULL; } @@ -200,7 +200,7 @@ static STSGroupBlockInfoEx* addOneGroupInfo(STSBuf* pTSBuf, int32_t id) { uint32_t newSize = (uint32_t)(pTSBuf->numOfAlloc * 1.5); assert((int32_t)newSize > pTSBuf->numOfAlloc); - STSGroupBlockInfoEx* tmp = (STSGroupBlockInfoEx*)realloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); + STSGroupBlockInfoEx* tmp = (STSGroupBlockInfoEx*)taosMemoryRealloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); if (tmp == NULL) { return NULL; } @@ -240,7 +240,7 @@ static STSGroupBlockInfoEx* addOneGroupInfo(STSBuf* pTSBuf, int32_t id) { static void shrinkBuffer(STSList* ptsData) { // shrink tmp buffer size if it consumes too many memory compared to the pre-defined size if (ptsData->allocSize >= ptsData->threshold * 2) { - char* rawBuf = realloc(ptsData->rawBuf, MEM_BUF_SIZE); + char* rawBuf = taosMemoryRealloc(ptsData->rawBuf, MEM_BUF_SIZE); if (rawBuf) { ptsData->rawBuf = rawBuf; ptsData->allocSize = MEM_BUF_SIZE; @@ -322,7 +322,7 @@ static void writeDataToDisk(STSBuf* pTSBuf) { static void expandBuffer(STSList* ptsData, int32_t inputSize) { if (ptsData->allocSize - ptsData->len < inputSize) { int32_t newSize = inputSize + ptsData->len; - char* tmp = realloc(ptsData->rawBuf, (size_t)newSize); + char* tmp = taosMemoryRealloc(ptsData->rawBuf, (size_t)newSize); if (tmp == NULL) { // todo } @@ -366,7 +366,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { // NOTE: mix types tags are not supported size_t sz = 0; if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { - char* tp = realloc(pBlock->tag.pz, pBlock->tag.nLen + 1); + char* tp = taosMemoryRealloc(pBlock->tag.pz, pBlock->tag.nLen + 1); assert(tp != NULL); memset(tp, 0, pBlock->tag.nLen + 1); @@ -812,7 +812,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { if (pDestBuf->numOfAlloc < newSize) { pDestBuf->numOfAlloc = newSize; - STSGroupBlockInfoEx* tmp = realloc(pDestBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); + STSGroupBlockInfoEx* tmp = taosMemoryRealloc(pDestBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); if (tmp == NULL) { return -1; } @@ -1028,13 +1028,13 @@ static STSBuf* allocResForTSBuf(STSBuf* pTSBuf) { const int32_t INITIAL_GROUPINFO_SIZE = 4; pTSBuf->numOfAlloc = INITIAL_GROUPINFO_SIZE; - pTSBuf->pData = calloc(pTSBuf->numOfAlloc, sizeof(STSGroupBlockInfoEx)); + pTSBuf->pData = taosMemoryCalloc(pTSBuf->numOfAlloc, sizeof(STSGroupBlockInfoEx)); if (pTSBuf->pData == NULL) { tsBufDestroy(pTSBuf); return NULL; } - pTSBuf->tsData.rawBuf = malloc(MEM_BUF_SIZE); + pTSBuf->tsData.rawBuf = taosMemoryMalloc(MEM_BUF_SIZE); if (pTSBuf->tsData.rawBuf == NULL) { tsBufDestroy(pTSBuf); return NULL; @@ -1044,13 +1044,13 @@ static STSBuf* allocResForTSBuf(STSBuf* pTSBuf) { pTSBuf->tsData.threshold = MEM_BUF_SIZE; pTSBuf->tsData.allocSize = MEM_BUF_SIZE; - pTSBuf->assistBuf = malloc(MEM_BUF_SIZE); + pTSBuf->assistBuf = taosMemoryMalloc(MEM_BUF_SIZE); if (pTSBuf->assistBuf == NULL) { tsBufDestroy(pTSBuf); return NULL; } - pTSBuf->block.payload = malloc(MEM_BUF_SIZE); + pTSBuf->block.payload = taosMemoryMalloc(MEM_BUF_SIZE); if (pTSBuf->block.payload == NULL) { tsBufDestroy(pTSBuf); return NULL; @@ -1079,7 +1079,7 @@ void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id) { return; } - (*id) = malloc(tsBufGetNumOfGroup(pTSBuf) * sizeof(int32_t)); + (*id) = taosMemoryMalloc(tsBufGetNumOfGroup(pTSBuf) * sizeof(int32_t)); for (int32_t i = 0; i < size; ++i) { (*id)[i] = pTSBuf->pData[i].info.id; diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index c6aa1cb81d7a7fe1c48f90f37218a14e63f7cf07..3995db89b615fa19a36924659e0af79cb78afdbc 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -199,14 +199,14 @@ void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uin case TSDB_DATA_TYPE_NCHAR: { // here we get the nchar length from raw binary bits length size_t lenInwchar = len / TSDB_NCHAR_SIZE; - pVar->ucs4 = calloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE); + pVar->ucs4 = taosMemoryCalloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE); memcpy(pVar->ucs4, pz, lenInwchar * TSDB_NCHAR_SIZE); pVar->nLen = (int32_t)len; break; } case TSDB_DATA_TYPE_BINARY: { // todo refactor, extract a method - pVar->pz = calloc(len + 1, sizeof(char)); + pVar->pz = taosMemoryCalloc(len + 1, sizeof(char)); memcpy(pVar->pz, pz, len); pVar->nLen = (int32_t)len; break; @@ -224,7 +224,7 @@ void taosVariantDestroy(SVariant *pVar) { if (pVar == NULL) return; if (pVar->nType == TSDB_DATA_TYPE_BINARY || pVar->nType == TSDB_DATA_TYPE_NCHAR) { - tfree(pVar->pz); + taosMemoryFreeClear(pVar->pz); pVar->nLen = 0; } @@ -233,7 +233,7 @@ void taosVariantDestroy(SVariant *pVar) { size_t num = taosArrayGetSize(pVar->arr); for (size_t i = 0; i < num; i++) { void *p = taosArrayGetP(pVar->arr, i); - free(p); + taosMemoryFree(p); } taosArrayDestroy(pVar->arr); pVar->arr = NULL; @@ -254,7 +254,7 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) { pDst->nType = pSrc->nType; if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR) { int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE; - char *p = realloc(pDst->pz, len); + char *p = taosMemoryRealloc(pDst->pz, len); assert(p); memset(p, 0, len); @@ -402,18 +402,18 @@ static int32_t toBinary(SVariant *pVariant, char **pDest, int32_t *pDestSize) { // it is a in-place convert type for SVariant, local buffer is needed if (*pDest == pVariant->pz) { - pBuf = calloc(1, INITIAL_ALLOC_SIZE); + pBuf = taosMemoryCalloc(1, INITIAL_ALLOC_SIZE); } if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) { size_t newSize = pVariant->nLen * TSDB_NCHAR_SIZE; if (pBuf != NULL) { if (newSize >= INITIAL_ALLOC_SIZE) { - pBuf = realloc(pBuf, newSize + 1); + pBuf = taosMemoryRealloc(pBuf, newSize + 1); } taosUcs4ToMbs(pVariant->ucs4, (int32_t)newSize, pBuf); - free(pVariant->ucs4); + taosMemoryFree(pVariant->ucs4); pBuf[newSize] = 0; } else { taosUcs4ToMbs(pVariant->ucs4, (int32_t)newSize, *pDest); @@ -460,23 +460,23 @@ static int32_t toNchar(SVariant *pVariant, char **pDest, int32_t *pDestSize) { } if (*pDest == pVariant->pz) { - TdUcs4 *pWStr = calloc(1, (nLen + 1) * TSDB_NCHAR_SIZE); + TdUcs4 *pWStr = taosMemoryCalloc(1, (nLen + 1) * TSDB_NCHAR_SIZE); bool ret = taosMbsToUcs4(pDst, nLen, pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL); if (!ret) { - tfree(pWStr); + taosMemoryFreeClear(pWStr); return -1; } // free the binary buffer in the first place if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { - free(pVariant->ucs4); + taosMemoryFree(pVariant->ucs4); } pVariant->ucs4 = pWStr; *pDestSize = taosUcs4len(pVariant->ucs4); // shrink the allocate memory, no need to check here. - char *tmp = realloc(pVariant->ucs4, (*pDestSize + 1) * TSDB_NCHAR_SIZE); + char *tmp = taosMemoryRealloc(pVariant->ucs4, (*pDestSize + 1) * TSDB_NCHAR_SIZE); assert(tmp != NULL); pVariant->ucs4 = (TdUcs4 *)tmp; @@ -526,7 +526,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result if (token.type == TK_NULL) { if (releaseVariantPtr) { - free(pVariant->pz); + taosMemoryFree(pVariant->pz); pVariant->nLen = 0; } @@ -547,7 +547,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result } if (releaseVariantPtr) { - free(pVariant->pz); + taosMemoryFree(pVariant->pz); pVariant->nLen = 0; } @@ -566,7 +566,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result if (token.type == TK_FLOAT) { double v = wcstod(pVariant->ucs4, &endPtr); if (releaseVariantPtr) { - free(pVariant->pz); + taosMemoryFree(pVariant->pz); pVariant->nLen = 0; } @@ -577,7 +577,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result *result = (int64_t)v; } else if (token.type == TK_NULL) { if (releaseVariantPtr) { - free(pVariant->pz); + taosMemoryFree(pVariant->pz); pVariant->nLen = 0; } setNull((char *)result, type, tDataTypes[type].bytes); @@ -585,7 +585,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result } else { int64_t val = wcstoll(pVariant->ucs4, &endPtr, 10); if (releaseVariantPtr) { - free(pVariant->pz); + taosMemoryFree(pVariant->pz); pVariant->nLen = 0; } @@ -971,21 +971,21 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) { errno = 0; double v = strtod(pVariant->pz, NULL); if ((errno == ERANGE && v == -1) || (isinf(v) || isnan(v))) { - free(pVariant->pz); + taosMemoryFree(pVariant->pz); return -1; } - free(pVariant->pz); + taosMemoryFree(pVariant->pz); pVariant->d = v; } else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) { errno = 0; double v = wcstod(pVariant->ucs4, NULL); if ((errno == ERANGE && v == -1) || (isinf(v) || isnan(v))) { - free(pVariant->pz); + taosMemoryFree(pVariant->pz); return -1; } - free(pVariant->pz); + taosMemoryFree(pVariant->pz); pVariant->d = v; } else if (pVariant->nType >= TSDB_DATA_TYPE_BOOL && pVariant->nType <= TSDB_DATA_TYPE_BIGINT) { double tmp = (double)pVariant->i; diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index d5fb61929a94109331e955e492cb3333b09c961a..ccd800d3f4686d70ffcc6da8c9951170cdd2aa03 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -99,7 +99,7 @@ TEST(testCase, toInteger_test) { } TEST(testCase, Datablock_test) { - SSDataBlock* b = static_cast(calloc(1, sizeof(SSDataBlock))); + SSDataBlock* b = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); b->info.numOfCols = 2; b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); @@ -108,8 +108,8 @@ TEST(testCase, Datablock_test) { infoData.info.type = TSDB_DATA_TYPE_INT; infoData.info.colId = 1; - infoData.pData = (char*) calloc(40, infoData.info.bytes); - infoData.nullbitmap = (char*) calloc(1, sizeof(char) * (40/8)); + infoData.pData = (char*) taosMemoryCalloc(40, infoData.info.bytes); + infoData.nullbitmap = (char*) taosMemoryCalloc(1, sizeof(char) * (40/8)); taosArrayPush(b->pDataBlock, &infoData); SColumnInfoData infoData1 = {0}; @@ -117,7 +117,7 @@ TEST(testCase, Datablock_test) { infoData1.info.type = TSDB_DATA_TYPE_BINARY; infoData1.info.colId = 2; - infoData1.varmeta.offset = (int32_t*) calloc(40, sizeof(uint32_t)); + infoData1.varmeta.offset = (int32_t*) taosMemoryCalloc(40, sizeof(uint32_t)); taosArrayPush(b->pDataBlock, &infoData1); char* str = "the value of: %d"; @@ -178,7 +178,7 @@ TEST(testCase, Datablock_test) { #if 0 TEST(testCase, non_var_dataBlock_split_test) { - SSDataBlock* b = static_cast(calloc(1, sizeof(SSDataBlock))); + SSDataBlock* b = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); b->info.numOfCols = 2; b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); @@ -189,8 +189,8 @@ TEST(testCase, non_var_dataBlock_split_test) { int32_t numOfRows = 1000000; - infoData.pData = (char*) calloc(numOfRows, infoData.info.bytes); - infoData.nullbitmap = (char*) calloc(1, sizeof(char) * (numOfRows/8)); + infoData.pData = (char*) taosMemoryCalloc(numOfRows, infoData.info.bytes); + infoData.nullbitmap = (char*) taosMemoryCalloc(1, sizeof(char) * (numOfRows/8)); taosArrayPush(b->pDataBlock, &infoData); SColumnInfoData infoData1 = {0}; @@ -198,8 +198,8 @@ TEST(testCase, non_var_dataBlock_split_test) { infoData1.info.type = TSDB_DATA_TYPE_TINYINT; infoData1.info.colId = 2; - infoData1.pData = (char*) calloc(numOfRows, infoData.info.bytes); - infoData1.nullbitmap = (char*) calloc(1, sizeof(char) * (numOfRows/8)); + infoData1.pData = (char*) taosMemoryCalloc(numOfRows, infoData.info.bytes); + infoData1.nullbitmap = (char*) taosMemoryCalloc(1, sizeof(char) * (numOfRows/8)); taosArrayPush(b->pDataBlock, &infoData1); for(int32_t i = 0; i < numOfRows; ++i) { @@ -233,7 +233,7 @@ TEST(testCase, non_var_dataBlock_split_test) { #endif TEST(testCase, var_dataBlock_split_test) { - SSDataBlock* b = static_cast(calloc(1, sizeof(SSDataBlock))); + SSDataBlock* b = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); b->info.numOfCols = 2; b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); @@ -244,8 +244,8 @@ TEST(testCase, var_dataBlock_split_test) { infoData.info.type = TSDB_DATA_TYPE_INT; infoData.info.colId = 1; - infoData.pData = (char*) calloc(numOfRows, infoData.info.bytes); - infoData.nullbitmap = (char*) calloc(1, sizeof(char) * (numOfRows/8)); + infoData.pData = (char*) taosMemoryCalloc(numOfRows, infoData.info.bytes); + infoData.nullbitmap = (char*) taosMemoryCalloc(1, sizeof(char) * (numOfRows/8)); taosArrayPush(b->pDataBlock, &infoData); SColumnInfoData infoData1 = {0}; @@ -253,7 +253,7 @@ TEST(testCase, var_dataBlock_split_test) { infoData1.info.type = TSDB_DATA_TYPE_BINARY; infoData1.info.colId = 2; - infoData1.varmeta.offset = (int32_t*) calloc(numOfRows, sizeof(uint32_t)); + infoData1.varmeta.offset = (int32_t*) taosMemoryCalloc(numOfRows, sizeof(uint32_t)); taosArrayPush(b->pDataBlock, &infoData1); char buf[41] = {0}; diff --git a/source/dnode/bnode/src/bnode.c b/source/dnode/bnode/src/bnode.c index 4236e85a7f4c28d5c93d7e5d0144c82bdad6f8fe..b9c41ebf4370fb070a39a7d76dab61742a174cf6 100644 --- a/source/dnode/bnode/src/bnode.c +++ b/source/dnode/bnode/src/bnode.c @@ -16,12 +16,12 @@ #include "bndInt.h" SBnode *bndOpen(const char *path, const SBnodeOpt *pOption) { - SBnode *pBnode = calloc(1, sizeof(SBnode)); + SBnode *pBnode = taosMemoryCalloc(1, sizeof(SBnode)); pBnode->msgCb = pOption->msgCb; return pBnode; } -void bndClose(SBnode *pBnode) { free(pBnode); } +void bndClose(SBnode *pBnode) { taosMemoryFree(pBnode); } int32_t bndGetLoad(SBnode *pBnode, SBnodeLoad *pLoad) { return 0; } diff --git a/source/dnode/mgmt/bnode/src/bmInt.c b/source/dnode/mgmt/bnode/src/bmInt.c index 34c6dbeb571338379e62919ef0bb2bd784a82948..e2506ab38322845918c269c66bbdddcb3c8c48d8 100644 --- a/source/dnode/mgmt/bnode/src/bmInt.c +++ b/source/dnode/mgmt/bnode/src/bmInt.c @@ -73,7 +73,7 @@ int32_t bmDrop(SMgmtWrapper *pWrapper) { bmCloseImp(pMgmt); taosRemoveDir(pMgmt->path); pWrapper->pMgmt = NULL; - free(pMgmt); + taosMemoryFree(pMgmt); dInfo("bnode-mgmt is dropped"); return 0; } @@ -85,13 +85,13 @@ static void bmClose(SMgmtWrapper *pWrapper) { dInfo("bnode-mgmt start to cleanup"); bmCloseImp(pMgmt); pWrapper->pMgmt = NULL; - free(pMgmt); + taosMemoryFree(pMgmt); dInfo("bnode-mgmt is cleaned up"); } int32_t bmOpen(SMgmtWrapper *pWrapper) { dInfo("bnode-mgmt start to init"); - SBnodeMgmt *pMgmt = calloc(1, sizeof(SBnodeMgmt)); + SBnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SBnodeMgmt)); if (pMgmt == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; diff --git a/source/dnode/mgmt/bnode/src/bmWorker.c b/source/dnode/mgmt/bnode/src/bmWorker.c index 2099787c0d31967c48cd6d75830fa300c4f3d11c..7698aa9dbdaaa2f8adbd24b9db2115dd142f16b9 100644 --- a/source/dnode/mgmt/bnode/src/bmWorker.c +++ b/source/dnode/mgmt/bnode/src/bmWorker.c @@ -77,7 +77,11 @@ int32_t bmStartWorker(SBnodeMgmt *pMgmt) { return -1; } + dDebug("bnode workers are initialized"); return 0; } -void bmStopWorker(SBnodeMgmt *pMgmt) { tMultiWorkerCleanup(&pMgmt->writeWorker); } +void bmStopWorker(SBnodeMgmt *pMgmt) { + tMultiWorkerCleanup(&pMgmt->writeWorker); + dDebug("bnode workers are closed"); +} diff --git a/source/dnode/mgmt/container/inc/dnd.h b/source/dnode/mgmt/container/inc/dnd.h index af552b53571094b5f683cb5f2739ba0c1dbf02c0..f6c8897f643c32d96c62334b673b4e4bdfe4d6b1 100644 --- a/source/dnode/mgmt/container/inc/dnd.h +++ b/source/dnode/mgmt/container/inc/dnd.h @@ -19,7 +19,6 @@ #include "os.h" #include "cJSON.h" -#include "monitor.h" #include "tcache.h" #include "tcrc32c.h" #include "tdatablock.h" @@ -36,8 +35,7 @@ #include "tworker.h" #include "dnode.h" -#include "tfs.h" -#include "wal.h" +#include "monitor.h" #ifdef __cplusplus extern "C" { @@ -53,7 +51,6 @@ extern "C" { typedef enum { DNODE, VNODES, QNODE, SNODE, MNODE, BNODE, NODE_MAX } ENodeType; typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EDndStatus; typedef enum { DND_ENV_INIT, DND_ENV_READY, DND_ENV_CLEANUP } EEnvStatus; -typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EWorkerType; typedef enum { PROC_SINGLE, PROC_CHILD, PROC_PARENT } EProcType; typedef struct SMgmtFp SMgmtFp; @@ -75,10 +72,8 @@ typedef int32_t (*DropNodeFp)(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); typedef int32_t (*RequireNodeFp)(SMgmtWrapper *pWrapper, bool *required); typedef struct SMsgHandle { - int32_t vgId; - NodeMsgFp vgIdMsgFp; - SMgmtWrapper *pVgIdWrapper; // Handle the case where the same message type is distributed to qnode or vnode - NodeMsgFp msgFp; + SMgmtWrapper *pQndWrapper; + SMgmtWrapper *pMndWrapper; SMgmtWrapper *pWrapper; } SMsgHandle; @@ -129,29 +124,30 @@ typedef struct SDnode { bool dropped; EDndStatus status; EDndEvent event; - EProcType procType; SStartupReq startup; TdFilePtr pLockFile; STransMgmt trans; SMgmtWrapper wrappers[NODE_MAX]; } SDnode; -EDndStatus dndGetStatus(SDnode *pDnode); -void dndSetStatus(SDnode *pDnode, EDndStatus stat); -SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType); -void dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp, int32_t vgId); -void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc); -void dndSendMonitorReport(SDnode *pDnode); +EDndStatus dndGetStatus(SDnode *pDnode); +void dndSetStatus(SDnode *pDnode, EDndStatus stat); +void dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp, int32_t vgId); +void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc); +void dndSendMonitorReport(SDnode *pDnode); int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, SEpSet *pEpSet, SRpcMsg *pMsg); void dndSendRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp); int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg); - int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed); int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed); +SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType); +int32_t dndMarkWrapper(SMgmtWrapper *pWrapper); +void dndReleaseWrapper(SMgmtWrapper *pWrapper); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mgmt/container/inc/dndInt.h b/source/dnode/mgmt/container/inc/dndInt.h index 9ae70874feff9e32630491271b0e42665f21b01e..d10835b67f760dce989290e806d815e8d83e18a5 100644 --- a/source/dnode/mgmt/container/inc/dndInt.h +++ b/source/dnode/mgmt/container/inc/dndInt.h @@ -34,7 +34,7 @@ int32_t dndInit(); void dndCleanup(); const char *dndStatStr(EDndStatus stat); void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup); -TdFilePtr dndCheckRunning(char *dataDir); +TdFilePtr dndCheckRunning(const char *dataDir); void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg); // dndMsg.c @@ -50,10 +50,6 @@ SDnode *dndCreate(const SDnodeOpt *pOption); void dndClose(SDnode *pDnode); void dndHandleEvent(SDnode *pDnode, EDndEvent event); -SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType); -int32_t dndMarkWrapper(SMgmtWrapper *pWrapper); -void dndReleaseWrapper(SMgmtWrapper *pWrapper); - // dndTransport.c int32_t dndInitServer(SDnode *pDnode); void dndCleanupServer(SDnode *pDnode); diff --git a/source/dnode/mgmt/container/src/dndExec.c b/source/dnode/mgmt/container/src/dndExec.c index d5c882398ede007bf6b047f2c4c7454b99c5ef73..3c29f80c94f260156bbb2badfd096a9361faa6d8 100644 --- a/source/dnode/mgmt/container/src/dndExec.c +++ b/source/dnode/mgmt/container/src/dndExec.c @@ -50,16 +50,23 @@ int32_t dndOpenNode(SMgmtWrapper *pWrapper) { } void dndCloseNode(SMgmtWrapper *pWrapper) { + dDebug("node:%s, start to close", pWrapper->name); taosWLockLatch(&pWrapper->latch); if (pWrapper->deployed) { (*pWrapper->fp.closeFp)(pWrapper); pWrapper->deployed = false; } + taosWUnLockLatch(&pWrapper->latch); + + while (pWrapper->refCount > 0) { + taosMsleep(10); + } + if (pWrapper->pProc) { taosProcCleanup(pWrapper->pProc); pWrapper->pProc = NULL; } - taosWUnLockLatch(&pWrapper->latch); + dDebug("node:%s, has been closed", pWrapper->name); } static int32_t dndRunInSingleProcess(SDnode *pDnode) { @@ -132,7 +139,7 @@ static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRsp, int32_t dTrace("msg:%p, get from parent queue", pRsp); pRsp->pCont = pCont; dndSendRpcRsp(pWrapper, pRsp); - free(pRsp); + taosMemoryFree(pRsp); } static int32_t dndRunInMultiProcess(SDnode *pDnode) { @@ -167,8 +174,8 @@ static int32_t dndRunInMultiProcess(SDnode *pDnode) { .childFreeBodyFp = (ProcFreeFp)rpcFreeCont, .parentQueueSize = 1024 * 1024 * 2, // size will be a configuration item .parentConsumeFp = (ProcConsumeFp)dndConsumeParentQueue, - .parentdMallocHeadFp = (ProcMallocFp)malloc, - .parentFreeHeadFp = (ProcFreeFp)free, + .parentdMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, + .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont, .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont, .testFlag = 0, diff --git a/source/dnode/mgmt/container/src/dndFile.c b/source/dnode/mgmt/container/src/dndFile.c index a5f8fb71690c86b355ea2d5d131ce20160d972b8..f860bf8e2de455f1e423965a74fe8c9f22bceb78 100644 --- a/source/dnode/mgmt/container/src/dndFile.c +++ b/source/dnode/mgmt/container/src/dndFile.c @@ -20,7 +20,7 @@ int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed) { int32_t code = TSDB_CODE_NODE_PARSE_FILE_ERROR; int32_t len = 0; int32_t maxLen = 1024; - char *content = calloc(1, maxLen + 1); + char *content = taosMemoryCalloc(1, maxLen + 1); cJSON *root = NULL; char file[PATH_MAX]; TdFilePtr pFile = NULL; @@ -57,7 +57,7 @@ int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed) { dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed); _OVER: - if (content != NULL) free(content); + if (content != NULL) taosMemoryFree(content); if (root != NULL) cJSON_Delete(root); if (pFile != NULL) taosCloseFile(&pFile); @@ -78,7 +78,7 @@ int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed) { int32_t len = 0; int32_t maxLen = 1024; - char *content = calloc(1, maxLen + 1); + char *content = taosMemoryCalloc(1, maxLen + 1); len += snprintf(content + len, maxLen - len, "{\n"); len += snprintf(content + len, maxLen - len, " \"deployed\": %d\n", deployed); @@ -87,7 +87,7 @@ int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed) { taosWriteFile(pFile, content, len); taosFsyncFile(pFile); taosCloseFile(&pFile); - free(content); + taosMemoryFree(content); char realfile[PATH_MAX]; snprintf(realfile, sizeof(realfile), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name); diff --git a/source/dnode/mgmt/container/src/dndInt.c b/source/dnode/mgmt/container/src/dndInt.c index 8ad4351a88f7fbd7255793b8eb23f986289caa69..ab19415168a573cc28c4d5d504d8a959c1f6bf00 100644 --- a/source/dnode/mgmt/container/src/dndInt.c +++ b/source/dnode/mgmt/container/src/dndInt.c @@ -15,11 +15,12 @@ #define _DEFAULT_SOURCE #include "dndInt.h" +#include "wal.h" static int8_t once = DND_ENV_INIT; int32_t dndInit() { - dDebug("start to init dnode env"); + dInfo("start to init dnode env"); if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { terrno = TSDB_CODE_REPEAT_INIT; dError("failed to init dnode env since %s", terrstr()); @@ -52,7 +53,7 @@ int32_t dndInit() { } void dndCleanup() { - dDebug("start to cleanup dnode env"); + dInfo("start to cleanup dnode env"); if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) { dError("dnode env is already cleaned up"); return; @@ -92,7 +93,7 @@ const char *dndStatStr(EDndStatus status) { } } -void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc) { +void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) { SStartupReq *pStartup = &pDnode->startup; tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); @@ -104,21 +105,21 @@ void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING); } -TdFilePtr dndCheckRunning(char *dataDir) { +TdFilePtr dndCheckRunning(const char *dataDir) { char filepath[PATH_MAX] = {0}; snprintf(filepath, sizeof(filepath), "%s/.running", dataDir); TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to lock file:%s since %s, quit", filepath, terrstr()); + dError("failed to lock file:%s since %s", filepath, terrstr()); return NULL; } int32_t ret = taosLockFile(pFile); if (ret != 0) { terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to lock file:%s since %s, quit", filepath, terrstr()); + dError("failed to lock file:%s since %s", filepath, terrstr()); taosCloseFile(&pFile); return NULL; } @@ -129,12 +130,10 @@ TdFilePtr dndCheckRunning(char *dataDir) { void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { dDebug("startup req is received"); - SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq)); dndGetStartup(pDnode, pStartup); dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); - SRpcMsg rpcRsp = {.handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq)}; rpcSendResponse(&rpcRsp); } diff --git a/source/dnode/mgmt/container/src/dndMonitor.c b/source/dnode/mgmt/container/src/dndMonitor.c index ef3db387de8deff0c26f0dac67f93c4740013089..c01f8407945ec3efa4193f9fb004dded3f559abb 100644 --- a/source/dnode/mgmt/container/src/dndMonitor.c +++ b/source/dnode/mgmt/container/src/dndMonitor.c @@ -22,7 +22,12 @@ static int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) { tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name)); pInfo->tempdir.size = tsTempSpace.size; - return vmMonitorTfsInfo(dndAcquireWrapper(pDnode, VNODES), pInfo); + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, VNODES); + if (pWrapper != NULL) { + vmMonitorTfsInfo(pWrapper, pInfo); + dndReleaseWrapper(pWrapper); + } + return 0; } static void dndGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { @@ -45,8 +50,17 @@ static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { taosGetCardInfo(&pInfo->net_in, &pInfo->net_out); taosGetProcIO(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk); - vmMonitorVnodeReqs(dndAcquireWrapper(pDnode, VNODES), pInfo); - pInfo->has_mnode = (dndAcquireWrapper(pDnode, MNODE)->required); + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, VNODES); + if (pWrapper != NULL) { + vmMonitorVnodeReqs(pWrapper, pInfo); + dndReleaseWrapper(pWrapper); + } + + pWrapper = dndAcquireWrapper(pDnode, MNODE); + if (pWrapper != NULL) { + pInfo->has_mnode = pWrapper->required; + dndReleaseWrapper(pWrapper); + } } void dndSendMonitorReport(SDnode *pDnode) { @@ -63,10 +77,15 @@ void dndSendMonitorReport(SDnode *pDnode) { SMonClusterInfo clusterInfo = {0}; SMonVgroupInfo vgroupInfo = {0}; SMonGrantInfo grantInfo = {0}; - if (mmMonitorMnodeInfo(dndAcquireWrapper(pDnode, MNODE), &clusterInfo, &vgroupInfo, &grantInfo) == 0) { - monSetClusterInfo(pMonitor, &clusterInfo); - monSetVgroupInfo(pMonitor, &vgroupInfo); - monSetGrantInfo(pMonitor, &grantInfo); + + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, MNODE); + if (pWrapper != NULL) { + if (mmMonitorMnodeInfo(pWrapper, &clusterInfo, &vgroupInfo, &grantInfo) == 0) { + monSetClusterInfo(pMonitor, &clusterInfo); + monSetVgroupInfo(pMonitor, &vgroupInfo); + monSetGrantInfo(pMonitor, &grantInfo); + } + dndReleaseWrapper(pWrapper); } SMonDnodeInfo dnodeInfo = {0}; diff --git a/source/dnode/mgmt/container/src/dndMsg.c b/source/dnode/mgmt/container/src/dndMsg.c index 21e9cc71a54ee790713dbc761440b0c435dd16fc..37ff4ebc0540058790ff720b5f2b5d1830aff71e 100644 --- a/source/dnode/mgmt/container/src/dndMsg.c +++ b/source/dnode/mgmt/container/src/dndMsg.c @@ -20,8 +20,8 @@ static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE); if (pWrapper != NULL) { dmUpdateMnodeEpSet(pWrapper->pMgmt, pEpSet); + dndReleaseWrapper(pWrapper); } - dndReleaseWrapper(pWrapper); } static inline NodeMsgFp dndGetMsgFp(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { diff --git a/source/dnode/mgmt/container/src/dndObj.c b/source/dnode/mgmt/container/src/dndObj.c index 75a06d585920735b336202c2db7d52e5fd11b0fd..ff414b5f5e684739b97562e82b30b2981ff8f4c1 100644 --- a/source/dnode/mgmt/container/src/dndObj.c +++ b/source/dnode/mgmt/container/src/dndObj.c @@ -39,29 +39,29 @@ static int32_t dndInitMemory(SDnode *pDnode, const SDnodeOpt *pOption) { static void dndClearMemory(SDnode *pDnode) { for (ENodeType n = 0; n < NODE_MAX; ++n) { SMgmtWrapper *pMgmt = &pDnode->wrappers[n]; - tfree(pMgmt->path); + taosMemoryFreeClear(pMgmt->path); } if (pDnode->pLockFile != NULL) { taosUnLockFile(pDnode->pLockFile); taosCloseFile(&pDnode->pLockFile); pDnode->pLockFile = NULL; } - tfree(pDnode->localEp); - tfree(pDnode->localFqdn); - tfree(pDnode->firstEp); - tfree(pDnode->secondEp); - tfree(pDnode->dataDir); - free(pDnode); + taosMemoryFreeClear(pDnode->localEp); + taosMemoryFreeClear(pDnode->localFqdn); + taosMemoryFreeClear(pDnode->firstEp); + taosMemoryFreeClear(pDnode->secondEp); + taosMemoryFreeClear(pDnode->dataDir); + taosMemoryFree(pDnode); dDebug("dnode object memory is cleared, data:%p", pDnode); } SDnode *dndCreate(const SDnodeOpt *pOption) { dInfo("start to create dnode object"); int32_t code = -1; - char path[PATH_MAX]; + char path[PATH_MAX] = {0}; SDnode *pDnode = NULL; - pDnode = calloc(1, sizeof(SDnode)); + pDnode = taosMemoryCalloc(1, sizeof(SDnode)); if (pDnode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; @@ -117,6 +117,7 @@ SDnode *dndCreate(const SDnodeOpt *pOption) { _OVER: if (code != 0 && pDnode) { dndClearMemory(pDnode); + pDnode = NULL; dError("failed to create dnode object since %s", terrstr()); } else { dInfo("dnode object is created, data:%p", pDnode); diff --git a/source/dnode/mgmt/container/src/dndTransport.c b/source/dnode/mgmt/container/src/dndTransport.c index 20069d89d63c54f2bdb3e3068c86f8e3febc5ad5..4acb1f459ef91cefc2b6e7cd7c7a3200ac663d85 100644 --- a/source/dnode/mgmt/container/src/dndTransport.c +++ b/source/dnode/mgmt/container/src/dndTransport.c @@ -20,13 +20,15 @@ #define INTERNAL_CKEY "_key" #define INTERNAL_SECRET "_pwd" -static inline void dndProcessQVnodeRpcMsg(SMsgHandle *pHandle, SRpcMsg *pMsg, SEpSet *pEpSet) { +static inline void dndProcessQMVnodeRpcMsg(SMsgHandle *pHandle, SRpcMsg *pMsg, SEpSet *pEpSet) { SMsgHead *pHead = pMsg->pCont; int32_t vgId = htonl(pHead->vgId); SMgmtWrapper *pWrapper = pHandle->pWrapper; - if (vgId == pHandle->vgId && pHandle->pVgIdWrapper != NULL) { - pWrapper = pHandle->pVgIdWrapper; + if (vgId == QND_VGID) { + pWrapper = pHandle->pQndWrapper; + } else if (vgId == MND_VGID) { + pWrapper = pHandle->pMndWrapper; } dTrace("msg:%s will be processed by %s, handle:%p app:%p vgId:%d", TMSG_INFO(pMsg->msgType), pWrapper->name, @@ -46,13 +48,13 @@ static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) { } SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)]; - if (pHandle->msgFp != NULL) { - if (pHandle->vgId == 0) { + if (pHandle->pWrapper != NULL) { + if (pHandle->pMndWrapper == NULL && pHandle->pQndWrapper == NULL) { dTrace("rsp:%s will be processed by %s, handle:%p app:%p code:0x%04x:%s", TMSG_INFO(msgType), pHandle->pWrapper->name, pRsp->handle, pRsp->ahandle, pRsp->code & 0XFFFF, tstrerror(pRsp->code)); dndProcessRpcMsg(pHandle->pWrapper, pRsp, pEpSet); } else { - dndProcessQVnodeRpcMsg(pHandle, pRsp, pEpSet); + dndProcessQMVnodeRpcMsg(pHandle, pRsp, pEpSet); } } else { dError("rsp:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pRsp->handle, pRsp->ahandle); @@ -126,13 +128,13 @@ static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) { } SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)]; - if (pHandle->msgFp != NULL) { - if (pHandle->vgId == 0) { + if (pHandle->pWrapper != NULL) { + if (pHandle->pMndWrapper == NULL && pHandle->pQndWrapper == NULL) { dTrace("req:%s will be processed by %s, handle:%p app:%p", TMSG_INFO(msgType), pHandle->pWrapper->name, pReq->handle, pReq->ahandle); dndProcessRpcMsg(pHandle->pWrapper, pReq, pEpSet); } else { - dndProcessQVnodeRpcMsg(pHandle, pReq, pEpSet); + dndProcessQMVnodeRpcMsg(pHandle, pReq, pEpSet); } } else { dError("req:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle); @@ -144,9 +146,14 @@ static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) { static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRpcRsp) { STransMgmt *pMgmt = &pDnode->trans; + SEpSet epSet = {0}; + + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE); + if (pWrapper != NULL) { + dmGetMnodeEpSet(pWrapper->pMgmt, &epSet); + dndReleaseWrapper(pWrapper); + } - SEpSet epSet = {0}; - dmGetMnodeEpSet(dndAcquireWrapper(pDnode, DNODE)->pMgmt, &epSet); rpcSendRecv(pMgmt->clientRpc, &epSet, pRpcMsg, pRpcRsp); } @@ -180,9 +187,14 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char return 0; } - if (mmGetUserAuth(dndAcquireWrapper(pDnode, MNODE), user, spi, encrypt, secret, ckey) == 0) { - dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); - return 0; + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, MNODE); + if (pWrapper != NULL) { + if (mmGetUserAuth(pWrapper, user, spi, encrypt, secret, ckey) == 0) { + dndReleaseWrapper(pWrapper); + dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); + return 0; + } + dndReleaseWrapper(pWrapper); } if (terrno != TSDB_CODE_APP_NOT_READY) { @@ -269,21 +281,27 @@ int32_t dndInitMsgHandle(SDnode *pDnode) { int32_t vgId = pWrapper->msgVgIds[msgIndex]; if (msgFp == NULL) continue; + // dTrace("msg:%s will be processed by %s, vgId:%d", tMsgInfo[msgIndex], pWrapper->name, vgId); + SMsgHandle *pHandle = &pMgmt->msgHandles[msgIndex]; - if (pHandle->msgFp != NULL && pHandle->vgId == vgId) { - dError("msg:%s has multiple process nodes, prev node:%s:%d, curr node:%s:%d", tMsgInfo[msgIndex], - pHandle->pWrapper->name, pHandle->pWrapper->msgVgIds[msgIndex], pWrapper->name, vgId); - return -1; + if (vgId == QND_VGID) { + if (pHandle->pQndWrapper != NULL) { + dError("msg:%s has multiple process nodes", tMsgInfo[msgIndex]); + return -1; + } + pHandle->pQndWrapper = pWrapper; + } else if (vgId == MND_VGID) { + if (pHandle->pMndWrapper != NULL) { + dError("msg:%s has multiple process nodes", tMsgInfo[msgIndex]); + return -1; + } + pHandle->pMndWrapper = pWrapper; } else { - dTrace("msg:%s will be processed by %s, vgId:%d", tMsgInfo[msgIndex], pWrapper->name, vgId); - if (vgId == 0) { - pHandle->msgFp = msgFp; - pHandle->pWrapper = pWrapper; - } else { - pHandle->vgId = vgId; - pHandle->vgIdMsgFp = msgFp; - pHandle->pVgIdWrapper = pWrapper; + if (pHandle->pWrapper != NULL) { + dError("msg:%s has multiple process nodes", tMsgInfo[msgIndex]); + return -1; } + pHandle->pWrapper = pWrapper; } } } @@ -320,7 +338,12 @@ int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pReq) { SDnode *pDnode = pWrapper->pDnode; STransMgmt *pTrans = &pDnode->trans; SEpSet epSet = {0}; - dmGetMnodeEpSet(dndAcquireWrapper(pDnode, DNODE)->pMgmt, &epSet); + + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE); + if (pWrapper != NULL) { + dmGetMnodeEpSet(pWrapper->pMgmt, &epSet); + dndReleaseWrapper(pWrapper); + } return dndSendRpcReq(pTrans, &epSet, pReq); } } @@ -328,7 +351,12 @@ int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pReq) { void dndSendRpcRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) { if (pRsp->code == TSDB_CODE_APP_NOT_READY) { SMgmtWrapper *pDnodeWrapper = dndAcquireWrapper(pWrapper->pDnode, DNODE); - dmSendRedirectRsp(pDnodeWrapper->pMgmt, pRsp); + if (pDnodeWrapper != NULL) { + dmSendRedirectRsp(pDnodeWrapper->pMgmt, pRsp); + dndReleaseWrapper(pDnodeWrapper); + } else { + rpcSendResponse(pRsp); + } } else { rpcSendResponse(pRsp); } diff --git a/source/dnode/mgmt/dnode/src/dmFile.c b/source/dnode/mgmt/dnode/src/dmFile.c index aa2247bcd5939f5268eaa141af2acc6f35474023..d44b1222a32f71a1273fb147166a59182aa65150 100644 --- a/source/dnode/mgmt/dnode/src/dmFile.c +++ b/source/dnode/mgmt/dnode/src/dmFile.c @@ -24,7 +24,7 @@ int32_t dmReadFile(SDnodeMgmt *pMgmt) { int32_t code = TSDB_CODE_NODE_PARSE_FILE_ERROR; int32_t len = 0; int32_t maxLen = 256 * 1024; - char *content = calloc(1, maxLen + 1); + char *content = taosMemoryCalloc(1, maxLen + 1); cJSON *root = NULL; char file[PATH_MAX]; TdFilePtr pFile = NULL; @@ -134,7 +134,7 @@ int32_t dmReadFile(SDnodeMgmt *pMgmt) { dmPrintDnodes(pMgmt); PRASE_DNODE_OVER: - if (content != NULL) free(content); + if (content != NULL) taosMemoryFree(content); if (root != NULL) cJSON_Delete(root); if (pFile != NULL) taosCloseFile(&pFile); @@ -171,7 +171,7 @@ int32_t dmWriteFile(SDnodeMgmt *pMgmt) { int32_t len = 0; int32_t maxLen = 256 * 1024; - char *content = calloc(1, maxLen + 1); + char *content = taosMemoryCalloc(1, maxLen + 1); len += snprintf(content + len, maxLen - len, "{\n"); len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", pDnode->dnodeId); @@ -197,7 +197,7 @@ int32_t dmWriteFile(SDnodeMgmt *pMgmt) { taosWriteFile(pFile, content, len); taosFsyncFile(pFile); taosCloseFile(&pFile); - free(content); + taosMemoryFree(content); char realfile[PATH_MAX]; snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP); @@ -209,7 +209,7 @@ int32_t dmWriteFile(SDnodeMgmt *pMgmt) { } pMgmt->updateTime = taosGetTimestampMs(); - dDebug("successed to write %s", file); + dDebug("successed to write %s", realfile); return 0; } diff --git a/source/dnode/mgmt/dnode/src/dmInt.c b/source/dnode/mgmt/dnode/src/dmInt.c index 1746cbe6e106a26eb7f8644f399ed8ce0b1b2b65..53049f7e7812bafc63e538b343250bb28371c560 100644 --- a/source/dnode/mgmt/dnode/src/dmInt.c +++ b/source/dnode/mgmt/dnode/src/dmInt.c @@ -74,14 +74,14 @@ void dmSendRedirectRsp(SDnodeMgmt *pMgmt, SRpcMsg *pReq) { } static int32_t dmStart(SMgmtWrapper *pWrapper) { - dDebug("dnode mgmt start to run"); + dDebug("dnode-mgmt start to run"); return dmStartThread(pWrapper->pMgmt); } int32_t dmInit(SMgmtWrapper *pWrapper) { SDnode *pDnode = pWrapper->pDnode; - SDnodeMgmt *pMgmt = calloc(1, sizeof(SDnodeMgmt)); - dInfo("dnode-mgmt is initialized"); + SDnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SDnodeMgmt)); + dInfo("dnode-mgmt start to init"); pDnode->dnodeId = 0; pDnode->dropped = 0; @@ -138,7 +138,7 @@ void dmCleanup(SMgmtWrapper *pWrapper) { taosWUnLockLatch(&pMgmt->latch); - free(pMgmt); + taosMemoryFree(pMgmt); pWrapper->pMgmt = NULL; dInfo("dnode-mgmt is cleaned up"); } diff --git a/source/dnode/mgmt/dnode/src/dmMsg.c b/source/dnode/mgmt/dnode/src/dmMsg.c index 7876d854ea0d879d619a4438d857e87a31eddbf4..836817e772a4a64fd541ca7e565d28b198143c9f 100644 --- a/source/dnode/mgmt/dnode/src/dmMsg.c +++ b/source/dnode/mgmt/dnode/src/dmMsg.c @@ -41,8 +41,12 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN); taosRUnLockLatch(&pMgmt->latch); - req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad)); - vmMonitorVnodeLoads(dndAcquireWrapper(pDnode, VNODES), req.pVloads); + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, VNODES); + if (pWrapper != NULL) { + req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad)); + vmMonitorVnodeLoads(pWrapper, req.pVloads); + dndReleaseWrapper(pWrapper); + } int32_t contLen = tSerializeSStatusReq(NULL, 0, &req); void *pHead = rpcMallocCont(contLen); @@ -114,19 +118,19 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { void dmInitMsgHandles(SMgmtWrapper *pWrapper) { // Requests handled by DNODE - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE, (NodeMsgFp)dmProcessMgmtMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE, (NodeMsgFp)dmProcessMgmtMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE, (NodeMsgFp)dmProcessMgmtMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE, (NodeMsgFp)dmProcessMgmtMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE, (NodeMsgFp)dmProcessMgmtMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE, (NodeMsgFp)dmProcessMgmtMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE, (NodeMsgFp)dmProcessMgmtMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE, (NodeMsgFp)dmProcessMgmtMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE, (NodeMsgFp)dmProcessMgmtMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_NETWORK_TEST, (NodeMsgFp)dmProcessMgmtMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_NETWORK_TEST, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); // Requests handled by MNODE - dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, (NodeMsgFp)dmProcessMgmtMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, (NodeMsgFp)dmProcessMgmtMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, (NodeMsgFp)dmProcessMgmtMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); } diff --git a/source/dnode/mgmt/dnode/src/dmWorker.c b/source/dnode/mgmt/dnode/src/dmWorker.c index b62c18655a01632807689b699e23e3e14ec3ff04..be08aad7cb271714e6cb32014419727d889fe9db 100644 --- a/source/dnode/mgmt/dnode/src/dmWorker.c +++ b/source/dnode/mgmt/dnode/src/dmWorker.c @@ -114,6 +114,7 @@ int32_t dmStartWorker(SDnodeMgmt *pMgmt) { return -1; } + dDebug("dnode workers are initialized"); return 0; } @@ -136,6 +137,7 @@ void dmStopWorker(SDnodeMgmt *pMgmt) { taosDestoryThread(pMgmt->threadId); pMgmt->threadId = NULL; } + dDebug("dnode workers are closed"); } int32_t dmProcessMgmtMsg(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { @@ -144,6 +146,6 @@ int32_t dmProcessMgmtMsg(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { pWorker = &pMgmt->statusWorker; } - dTrace("msg:%p, will be written to worker %s", pMsg, pWorker->name); + dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); return taosWriteQitem(pWorker->queue, pMsg); } diff --git a/source/dnode/mgmt/mnode/src/mmFile.c b/source/dnode/mgmt/mnode/src/mmFile.c index 757b39747390076536279eb5311227a265e9c397..e5cc0ce0871bfa398bd7c46f4c9260bb16e8f0de 100644 --- a/source/dnode/mgmt/mnode/src/mmFile.c +++ b/source/dnode/mgmt/mnode/src/mmFile.c @@ -20,7 +20,7 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, bool *pDeployed) { int32_t code = TSDB_CODE_NODE_PARSE_FILE_ERROR; int32_t len = 0; int32_t maxLen = 4096; - char *content = calloc(1, maxLen + 1); + char *content = taosMemoryCalloc(1, maxLen + 1); cJSON *root = NULL; char file[PATH_MAX]; TdFilePtr pFile = NULL; @@ -97,7 +97,7 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, bool *pDeployed) { dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed); PRASE_MNODE_OVER: - if (content != NULL) free(content); + if (content != NULL) taosMemoryFree(content); if (root != NULL) cJSON_Delete(root); if (pFile != NULL) taosCloseFile(&pFile); @@ -118,7 +118,7 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, bool deployed) { int32_t len = 0; int32_t maxLen = 4096; - char *content = calloc(1, maxLen + 1); + char *content = taosMemoryCalloc(1, maxLen + 1); len += snprintf(content + len, maxLen - len, "{\n"); len += snprintf(content + len, maxLen - len, " \"deployed\": %d,\n", deployed); @@ -139,7 +139,7 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, bool deployed) { taosWriteFile(pFile, content, len); taosFsyncFile(pFile); taosCloseFile(&pFile); - free(content); + taosMemoryFree(content); char realfile[PATH_MAX]; snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP); diff --git a/source/dnode/mgmt/mnode/src/mmInt.c b/source/dnode/mgmt/mnode/src/mmInt.c index aa348ae87947b99f189a9ab50fec41c1507652ea..cefa36ef10a816c3cffb11df8b2d0ea9fb574b96 100644 --- a/source/dnode/mgmt/mnode/src/mmInt.c +++ b/source/dnode/mgmt/mnode/src/mmInt.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mmInt.h" +#include "wal.h" static bool mmDeployRequired(SDnode *pDnode) { if (pDnode->dnodeId > 0) return false; @@ -176,7 +177,7 @@ int32_t mmDrop(SMgmtWrapper *pWrapper) { mmCloseImp(pMgmt); taosRemoveDir(pMgmt->path); pWrapper->pMgmt = NULL; - free(pMgmt); + taosMemoryFree(pMgmt); dInfo("mnode-mgmt is dropped"); return 0; } @@ -188,7 +189,7 @@ static void mmClose(SMgmtWrapper *pWrapper) { dInfo("mnode-mgmt start to cleanup"); mmCloseImp(pMgmt); pWrapper->pMgmt = NULL; - free(pMgmt); + taosMemoryFree(pMgmt); dInfo("mnode-mgmt is cleaned up"); } @@ -199,7 +200,7 @@ int32_t mmOpenFromMsg(SMgmtWrapper *pWrapper, SDCreateMnodeReq *pReq) { return -1; } - SMnodeMgmt *pMgmt = calloc(1, sizeof(SMnodeMgmt)); + SMnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SMnodeMgmt)); if (pMgmt == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -226,7 +227,7 @@ static int32_t mmOpen(SMgmtWrapper *pWrapper) { } static int32_t mmStart(SMgmtWrapper *pWrapper) { - dDebug("mnode mgmt start to run"); + dDebug("mnode-mgmt start to run"); SMnodeMgmt *pMgmt = pWrapper->pMgmt; return mndStart(pMgmt->pMnode); } diff --git a/source/dnode/mgmt/mnode/src/mmMsg.c b/source/dnode/mgmt/mnode/src/mmMsg.c index 56a580ed939df45a625a4bcb54bcca611fb9d797..1cae2220ad4edbc69fe8c91b3f456d802bcc9321 100644 --- a/source/dnode/mgmt/mnode/src/mmMsg.c +++ b/source/dnode/mgmt/mnode/src/mmMsg.c @@ -75,78 +75,90 @@ int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { void mmInitMsgHandles(SMgmtWrapper *pWrapper) { // Requests handled by DNODE - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); // Requests handled by MNODE - dndSetMsgHandle(pWrapper, TDMT_MND_CONNECT, (NodeMsgFp)mmProcessReadMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_ACCT, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_ACCT, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_ACCT, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_USER, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_USER, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_USER, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_GET_USER_AUTH, (NodeMsgFp)mmProcessReadMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DNODE, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_CONFIG_DNODE, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DNODE, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_MNODE, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_BNODE, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DB, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DB, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_USE_DB, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_DB, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_SYNC_DB, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_COMPACT_DB, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_FUNC, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_RETRIEVE_FUNC, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_FUNC, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STB, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_STB, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_STB, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_TABLE_META, (NodeMsgFp)mmProcessReadMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_VGROUP_LIST, (NodeMsgFp)mmProcessReadMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_KILL_QUERY, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_KILL_CONN, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_HEARTBEAT, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_SHOW, (NodeMsgFp)mmProcessReadMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_SHOW_RETRIEVE, (NodeMsgFp)mmProcessReadMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_SYSTABLE_RETRIEVE, (NodeMsgFp)mmProcessReadMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_STATUS, (NodeMsgFp)mmProcessReadMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_KILL_TRANS, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_GRANT, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_AUTH, (NodeMsgFp)mmProcessReadMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_TOPIC, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_TOPIC, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, (NodeMsgFp)mmProcessReadMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_CONNECT, (NodeMsgFp)mmProcessReadMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_ACCT, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_ACCT, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_ACCT, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_USER, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_USER, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_USER, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_GET_USER_AUTH, (NodeMsgFp)mmProcessReadMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CONFIG_DNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_MNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_BNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_USE_DB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_DB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_SYNC_DB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_COMPACT_DB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_FUNC, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_RETRIEVE_FUNC, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_FUNC, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_STB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_STB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SMA, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SMA, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_TABLE_META, (NodeMsgFp)mmProcessReadMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_VGROUP_LIST, (NodeMsgFp)mmProcessReadMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_QUERY, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_CONN, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_HEARTBEAT, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_SHOW, (NodeMsgFp)mmProcessReadMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_SHOW_RETRIEVE, (NodeMsgFp)mmProcessReadMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_SYSTABLE_RETRIEVE, (NodeMsgFp)mmProcessReadMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_STATUS, (NodeMsgFp)mmProcessReadMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_TRANS, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_GRANT, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_AUTH, (NodeMsgFp)mmProcessReadMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_TOPIC, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_TOPIC, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, (NodeMsgFp)mmProcessReadMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); // Requests handled by VNODE - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); + + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)mmProcessReadMsg, MND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)mmProcessReadMsg, MND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)mmProcessReadMsg, MND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)mmProcessReadMsg, MND_VGID); + } diff --git a/source/dnode/mgmt/mnode/src/mmWorker.c b/source/dnode/mgmt/mnode/src/mmWorker.c index b9a3a4f14cc58340766cb2eec94ff63741d8955f..d6b150106da2808b335dae892aeac1f1f66f960b 100644 --- a/source/dnode/mgmt/mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mnode/src/mmWorker.c @@ -108,6 +108,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { return -1; } + dDebug("mnode workers are initialized"); return 0; } @@ -115,4 +116,5 @@ void mmStopWorker(SMnodeMgmt *pMgmt) { tSingleWorkerCleanup(&pMgmt->readWorker); tSingleWorkerCleanup(&pMgmt->writeWorker); tSingleWorkerCleanup(&pMgmt->syncWorker); + dDebug("mnode workers are closed"); } diff --git a/source/dnode/mgmt/qnode/src/qmInt.c b/source/dnode/mgmt/qnode/src/qmInt.c index 2bbfbd83f5c03461e23161b69711a719afcc1073..c8cb7258c3873f9267244877caacddf0b6d6548b 100644 --- a/source/dnode/mgmt/qnode/src/qmInt.c +++ b/source/dnode/mgmt/qnode/src/qmInt.c @@ -76,7 +76,7 @@ int32_t qmDrop(SMgmtWrapper *pWrapper) { qmCloseImp(pMgmt); taosRemoveDir(pMgmt->path); pWrapper->pMgmt = NULL; - free(pMgmt); + taosMemoryFree(pMgmt); dInfo("qnode-mgmt is dropped"); return 0; } @@ -88,13 +88,13 @@ static void qmClose(SMgmtWrapper *pWrapper) { dInfo("qnode-mgmt start to cleanup"); qmCloseImp(pMgmt); pWrapper->pMgmt = NULL; - free(pMgmt); + taosMemoryFree(pMgmt); dInfo("qnode-mgmt is cleaned up"); } int32_t qmOpen(SMgmtWrapper *pWrapper) { dInfo("qnode-mgmt start to init"); - SQnodeMgmt *pMgmt = calloc(1, sizeof(SQnodeMgmt)); + SQnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SQnodeMgmt)); if (pMgmt == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; diff --git a/source/dnode/mgmt/qnode/src/qmMsg.c b/source/dnode/mgmt/qnode/src/qmMsg.c index ddcd791b7bfa55b0a8e8b73d5f45b5e899c150ff..ebe6477e8187e29bdb97181d2f3e886f968c2db3 100644 --- a/source/dnode/mgmt/qnode/src/qmMsg.c +++ b/source/dnode/mgmt/qnode/src/qmMsg.c @@ -56,14 +56,14 @@ int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { void qmInitMsgHandles(SMgmtWrapper *pWrapper) { // Requests handled by VNODE - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)qmProcessQueryMsg, 1); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)qmProcessQueryMsg, 1); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)qmProcessFetchMsg, 1); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)qmProcessFetchMsg, 1); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)qmProcessQueryMsg, QND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)qmProcessQueryMsg, QND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)qmProcessFetchMsg, QND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)qmProcessFetchMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)qmProcessFetchMsg, 1); - dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)qmProcessFetchMsg, 1); - dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)qmProcessFetchMsg, 1); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)qmProcessFetchMsg, 1); - dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)qmProcessFetchMsg, 1); + dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)qmProcessFetchMsg, QND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)qmProcessFetchMsg, QND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)qmProcessFetchMsg, QND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)qmProcessFetchMsg, QND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)qmProcessFetchMsg, QND_VGID); } diff --git a/source/dnode/mgmt/qnode/src/qmWorker.c b/source/dnode/mgmt/qnode/src/qmWorker.c index fff469a9020b1ef27dbdade86f249d47f16fc659..aa4da8279074485f29a2afdfbce2291e13d3933d 100644 --- a/source/dnode/mgmt/qnode/src/qmWorker.c +++ b/source/dnode/mgmt/qnode/src/qmWorker.c @@ -132,10 +132,12 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) { return -1; } + dDebug("qnode workers are initialized"); return 0; } void qmStopWorker(SQnodeMgmt *pMgmt) { tSingleWorkerCleanup(&pMgmt->queryWorker); tSingleWorkerCleanup(&pMgmt->fetchWorker); + dDebug("qnode workers are closed"); } diff --git a/source/dnode/mgmt/snode/src/smInt.c b/source/dnode/mgmt/snode/src/smInt.c index b4d6576e5740d40388f6d5d6a04efc8aa1316669..351f7d656eedd61fee3faa555caa377e60df8974 100644 --- a/source/dnode/mgmt/snode/src/smInt.c +++ b/source/dnode/mgmt/snode/src/smInt.c @@ -73,7 +73,7 @@ int32_t smDrop(SMgmtWrapper *pWrapper) { smCloseImp(pMgmt); taosRemoveDir(pMgmt->path); pWrapper->pMgmt = NULL; - free(pMgmt); + taosMemoryFree(pMgmt); dInfo("snode-mgmt is dropped"); return 0; } @@ -85,13 +85,13 @@ static void smClose(SMgmtWrapper *pWrapper) { dInfo("snode-mgmt start to cleanup"); smCloseImp(pMgmt); pWrapper->pMgmt = NULL; - free(pMgmt); + taosMemoryFree(pMgmt); dInfo("snode-mgmt is cleaned up"); } int32_t smOpen(SMgmtWrapper *pWrapper) { dInfo("snode-mgmt start to init"); - SSnodeMgmt *pMgmt = calloc(1, sizeof(SSnodeMgmt)); + SSnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SSnodeMgmt)); if (pMgmt == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; diff --git a/source/dnode/mgmt/snode/src/smMsg.c b/source/dnode/mgmt/snode/src/smMsg.c index 09865562078b84d692a57c26d414fa3ef27906a4..aea1dded5679bacc9e6fd5fc2dcf01295acad99a 100644 --- a/source/dnode/mgmt/snode/src/smMsg.c +++ b/source/dnode/mgmt/snode/src/smMsg.c @@ -56,6 +56,6 @@ int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { void smInitMsgHandles(SMgmtWrapper *pWrapper) { // Requests handled by SNODE - dndSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, (NodeMsgFp)smProcessMgmtMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, (NodeMsgFp)smProcessExecMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, (NodeMsgFp)smProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, (NodeMsgFp)smProcessExecMsg, VND_VGID); } diff --git a/source/dnode/mgmt/snode/src/smWorker.c b/source/dnode/mgmt/snode/src/smWorker.c index ceec6b82c330ae023febe5d86673c7a870cdbc95..5913713ff3bb6ffb0d291a27cec85d6dabee32e8 100644 --- a/source/dnode/mgmt/snode/src/smWorker.c +++ b/source/dnode/mgmt/snode/src/smWorker.c @@ -51,7 +51,7 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) { } for (int32_t i = 0; i < SND_UNIQUE_THREAD_NUM; i++) { - SMultiWorker *pUniqueWorker = malloc(sizeof(SMultiWorker)); + SMultiWorker *pUniqueWorker = taosMemoryMalloc(sizeof(SMultiWorker)); if (pUniqueWorker == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -80,6 +80,7 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) { return -1; } + dDebug("snode workers are initialized"); return 0; } @@ -90,18 +91,20 @@ void smStopWorker(SSnodeMgmt *pMgmt) { } taosArrayDestroy(pMgmt->uniqueWorkers); tSingleWorkerCleanup(&pMgmt->sharedWorker); + dDebug("snode workers are closed"); } static FORCE_INLINE int32_t smGetSWIdFromMsg(SRpcMsg *pMsg) { SMsgHead *pHead = pMsg->pCont; - pHead->streamTaskId = htonl(pHead->streamTaskId); - return pHead->streamTaskId % SND_UNIQUE_THREAD_NUM; + pHead->vgId = htonl(pHead->vgId); + return pHead->vgId % SND_UNIQUE_THREAD_NUM; } static FORCE_INLINE int32_t smGetSWTypeFromMsg(SRpcMsg *pMsg) { - SStreamExecMsgHead *pHead = pMsg->pCont; - pHead->workerType = htonl(pHead->workerType); - return pHead->workerType; + /*SMsgHead *pHead = pMsg->pCont;*/ + /*pHead->workerType = htonl(pHead->workerType);*/ + /*return pHead->workerType;*/ + return 0; } int32_t smProcessMgmtMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { diff --git a/source/dnode/mgmt/test/sut/inc/sut.h b/source/dnode/mgmt/test/sut/inc/sut.h index 304a370bcd4af4b5dda70b5131f9aadc95994ffb..f7e2831160519e18fa2bd8d1d0885922a41efdae 100644 --- a/source/dnode/mgmt/test/sut/inc/sut.h +++ b/source/dnode/mgmt/test/sut/inc/sut.h @@ -98,7 +98,7 @@ class Testbase { #define CheckBinaryByte(b, len) \ { \ - char* bytes = (char*)calloc(1, len); \ + char* bytes = (char*)taosMemoryCalloc(1, len); \ for (int32_t i = 0; i < len - 1; ++i) { \ bytes[i] = b; \ } \ diff --git a/source/dnode/mgmt/test/sut/src/client.cpp b/source/dnode/mgmt/test/sut/src/client.cpp index f22bc9d27696a270c9913acc44ce97fe90795e6c..a1165d5bc9170345402483dc24985700fa65f535 100644 --- a/source/dnode/mgmt/test/sut/src/client.cpp +++ b/source/dnode/mgmt/test/sut/src/client.cpp @@ -25,9 +25,9 @@ static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) { void TestClient::SetRpcRsp(SRpcMsg* rsp) { if (this->pRsp) { - free(this->pRsp); + taosMemoryFree(this->pRsp); } - this->pRsp = (SRpcMsg*)calloc(1, sizeof(SRpcMsg)); + this->pRsp = (SRpcMsg*)taosMemoryCalloc(1, sizeof(SRpcMsg)); this->pRsp->msgType = rsp->msgType; this->pRsp->code = rsp->code; this->pRsp->pCont = rsp->pCont; diff --git a/source/dnode/mgmt/test/vnode/vnode.cpp b/source/dnode/mgmt/test/vnode/vnode.cpp index 4457faf7b1aa97215fa9433385e9e0438f8cd5da..e2de3fc33ae473b76cfea9c83ccfb4e03d2e8f82 100644 --- a/source/dnode/mgmt/test/vnode/vnode.cpp +++ b/source/dnode/mgmt/test/vnode/vnode.cpp @@ -158,6 +158,7 @@ TEST_F(DndTestVnode, 03_Create_Stb) { for (int i = 0; i < 1; ++i) { SVCreateTbReq req = {0}; req.ver = 0; + req.dbFName = (char*)"1.db1"; req.name = (char*)"stb1"; req.ttl = 0; req.keep = 0; @@ -229,6 +230,7 @@ TEST_F(DndTestVnode, 04_Alter_Stb) { for (int i = 0; i < 1; ++i) { SVCreateTbReq req = {0}; req.ver = 0; + req.dbFName = (char*)"1.db1"; req.name = (char*)"stb1"; req.ttl = 0; req.keep = 0; diff --git a/source/dnode/mgmt/vnode/inc/vmInt.h b/source/dnode/mgmt/vnode/inc/vmInt.h index ccdb1ae257d7ef8030a09d401c8263c0a4833ca1..197c606a0d582b28ea57755452d9336326f522ed 100644 --- a/source/dnode/mgmt/vnode/inc/vmInt.h +++ b/source/dnode/mgmt/vnode/inc/vmInt.h @@ -33,6 +33,7 @@ typedef struct SVnodesMgmt { SQWorkerPool fetchPool; SWWorkerPool syncPool; SWWorkerPool writePool; + SWWorkerPool mergePool; const char *path; SDnode *pDnode; SMgmtWrapper *pWrapper; @@ -63,6 +64,7 @@ typedef struct { STaosQueue *pApplyQ; STaosQueue *pQueryQ; STaosQueue *pFetchQ; + STaosQueue *pMergeQ; SMgmtWrapper *pWrapper; } SVnodeObj; @@ -110,10 +112,11 @@ int32_t vmProcessWriteMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); int32_t vmProcessSyncMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); int32_t vmProcessQueryMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); int32_t vmProcessFetchMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); +int32_t vmProcessMergeMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); int32_t vmProcessMgmtMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); #ifdef __cplusplus } #endif -#endif /*_TD_DND_VNODES_INT_H_*/ \ No newline at end of file +#endif /*_TD_DND_VNODES_INT_H_*/ diff --git a/source/dnode/mgmt/vnode/src/vmFile.c b/source/dnode/mgmt/vnode/src/vmFile.c index 0fe868dfa41247c0a34ff293d81f11a93c182a01..d4f906b482c8c2f21e2fa84ea96b91bb3ecf3348 100644 --- a/source/dnode/mgmt/vnode/src/vmFile.c +++ b/source/dnode/mgmt/vnode/src/vmFile.c @@ -21,7 +21,7 @@ SVnodeObj **vmGetVnodesFromHash(SVnodesMgmt *pMgmt, int32_t *numOfVnodes) { int32_t num = 0; int32_t size = taosHashGetSize(pMgmt->hash); - SVnodeObj **pVnodes = calloc(size, sizeof(SVnodeObj *)); + SVnodeObj **pVnodes = taosMemoryCalloc(size, sizeof(SVnodeObj *)); void *pIter = taosHashIterate(pMgmt->hash, NULL); while (pIter) { @@ -48,7 +48,7 @@ int32_t vmGetVnodesFromFile(SVnodesMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *n int32_t code = TSDB_CODE_NODE_PARSE_FILE_ERROR; int32_t len = 0; int32_t maxLen = 30000; - char *content = calloc(1, maxLen + 1); + char *content = taosMemoryCalloc(1, maxLen + 1); cJSON *root = NULL; FILE *fp = NULL; char file[PATH_MAX]; @@ -85,7 +85,7 @@ int32_t vmGetVnodesFromFile(SVnodesMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *n int32_t vnodesNum = cJSON_GetArraySize(vnodes); if (vnodesNum > 0) { - pCfgs = calloc(vnodesNum, sizeof(SWrapperCfg)); + pCfgs = taosMemoryCalloc(vnodesNum, sizeof(SWrapperCfg)); if (pCfgs == NULL) { dError("failed to read %s since out of memory", file); goto PRASE_VNODE_OVER; @@ -140,7 +140,7 @@ int32_t vmGetVnodesFromFile(SVnodesMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *n dInfo("succcessed to read file %s", file); PRASE_VNODE_OVER: - if (content != NULL) free(content); + if (content != NULL) taosMemoryFree(content); if (root != NULL) cJSON_Delete(root); if (pFile != NULL) taosCloseFile(&pFile); @@ -166,7 +166,7 @@ int32_t vmWriteVnodesToFile(SVnodesMgmt *pMgmt) { int32_t len = 0; int32_t maxLen = 65536; - char *content = calloc(1, maxLen + 1); + char *content = taosMemoryCalloc(1, maxLen + 1); len += snprintf(content + len, maxLen - len, "{\n"); len += snprintf(content + len, maxLen - len, " \"vnodes\": [\n"); @@ -190,7 +190,7 @@ int32_t vmWriteVnodesToFile(SVnodesMgmt *pMgmt) { taosWriteFile(pFile, content, len); taosFsyncFile(pFile); taosCloseFile(&pFile); - free(content); + taosMemoryFree(content); terrno = 0; for (int32_t i = 0; i < numOfVnodes; ++i) { @@ -199,7 +199,7 @@ int32_t vmWriteVnodesToFile(SVnodesMgmt *pMgmt) { } if (pVnodes != NULL) { - free(pVnodes); + taosMemoryFree(pVnodes); } dDebug("successed to write %s", realfile); diff --git a/source/dnode/mgmt/vnode/src/vmInt.c b/source/dnode/mgmt/vnode/src/vmInt.c index 746fcd4855c1e43a4ffca150c5ae0427abe4a86b..464e789e468bc496e8680649b63267db8768ac3e 100644 --- a/source/dnode/mgmt/vnode/src/vmInt.c +++ b/source/dnode/mgmt/vnode/src/vmInt.c @@ -46,7 +46,7 @@ void vmReleaseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { } int32_t vmOpenVnode(SVnodesMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { - SVnodeObj *pVnode = calloc(1, sizeof(SVnodeObj)); + SVnodeObj *pVnode = taosMemoryCalloc(1, sizeof(SVnodeObj)); if (pVnode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -107,9 +107,9 @@ void vmCloseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { vnodeDestroy(pVnode->path); } - free(pVnode->path); - free(pVnode->db); - free(pVnode); + taosMemoryFree(pVnode->path); + taosMemoryFree(pVnode->db); + taosMemoryFree(pVnode); } static void *vmOpenVnodeFunc(void *param) { @@ -183,11 +183,11 @@ static int32_t vmOpenVnodes(SVnodesMgmt *pMgmt) { #endif int32_t vnodesPerThread = numOfVnodes / threadNum + 1; - SVnodeThread *threads = calloc(threadNum, sizeof(SVnodeThread)); + SVnodeThread *threads = taosMemoryCalloc(threadNum, sizeof(SVnodeThread)); for (int32_t t = 0; t < threadNum; ++t) { threads[t].threadIndex = t; threads[t].pMgmt = pMgmt; - threads[t].pCfgs = calloc(vnodesPerThread, sizeof(SWrapperCfg)); + threads[t].pCfgs = taosMemoryCalloc(vnodesPerThread, sizeof(SWrapperCfg)); } for (int32_t v = 0; v < numOfVnodes; ++v) { @@ -217,10 +217,10 @@ static int32_t vmOpenVnodes(SVnodesMgmt *pMgmt) { if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) { taosThreadJoin(pThread->thread, NULL); } - free(pThread->pCfgs); + taosMemoryFree(pThread->pCfgs); } - free(threads); - free(pCfgs); + taosMemoryFree(threads); + taosMemoryFree(pCfgs); if (pMgmt->state.openVnodes != pMgmt->state.totalVnodes) { dError("there are total vnodes:%d, opened:%d", pMgmt->state.totalVnodes, pMgmt->state.openVnodes); @@ -242,7 +242,7 @@ static void vmCloseVnodes(SVnodesMgmt *pMgmt) { } if (pVnodes != NULL) { - free(pVnodes); + taosMemoryFree(pVnodes); } if (pMgmt->hash != NULL) { @@ -257,22 +257,22 @@ static void vmCleanup(SMgmtWrapper *pWrapper) { SVnodesMgmt *pMgmt = pWrapper->pMgmt; if (pMgmt == NULL) return; - dInfo("vnodes-mgmt start to cleanup"); + dInfo("vnode-mgmt start to cleanup"); vmCloseVnodes(pMgmt); vmStopWorker(pMgmt); vnodeCleanup(); // walCleanUp(); - free(pMgmt); + taosMemoryFree(pMgmt); pWrapper->pMgmt = NULL; - dInfo("vnodes-mgmt is cleaned up"); + dInfo("vnode-mgmt is cleaned up"); } static int32_t vmInit(SMgmtWrapper *pWrapper) { SDnode *pDnode = pWrapper->pDnode; - SVnodesMgmt *pMgmt = calloc(1, sizeof(SVnodesMgmt)); + SVnodesMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SVnodesMgmt)); int32_t code = -1; - dInfo("vnodes-mgmt start to init"); + dInfo("vnode-mgmt start to init"); if (pMgmt == NULL) goto _OVER; pMgmt->path = pWrapper->path; @@ -312,7 +312,7 @@ static int32_t vmInit(SMgmtWrapper *pWrapper) { } if (vmOpenVnodes(pMgmt) != 0) { - dError("failed to open vnodes since %s", terrstr()); + dError("failed to open vnode since %s", terrstr()); return -1; } diff --git a/source/dnode/mgmt/vnode/src/vmMsg.c b/source/dnode/mgmt/vnode/src/vmMsg.c index d4af82b382fb5c1ee42d0b6534961f40eb7232dd..97d829571fe4438e46b59c4a677d296b789d9b81 100644 --- a/source/dnode/mgmt/vnode/src/vmMsg.c +++ b/source/dnode/mgmt/vnode/src/vmMsg.c @@ -244,43 +244,49 @@ int32_t vmProcessCompactVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { void vmInitMsgHandles(SMgmtWrapper *pWrapper) { // Requests handled by VNODE - dndSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, (NodeMsgFp)vmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)vmProcessQueryMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)vmProcessQueryMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)vmProcessFetchMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)vmProcessFetchMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, (NodeMsgFp)vmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, (NodeMsgFp)vmProcessFetchMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_TABLES_META, (NodeMsgFp)vmProcessFetchMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONSUME, (NodeMsgFp)vmProcessQueryMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, (NodeMsgFp)vmProcessQueryMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, (NodeMsgFp)vmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, (NodeMsgFp)vmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)vmProcessFetchMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)vmProcessFetchMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)vmProcessFetchMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)vmProcessFetchMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB, (NodeMsgFp)vmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB, (NodeMsgFp)vmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB, (NodeMsgFp)vmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, (NodeMsgFp)vmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, (NodeMsgFp)vmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)vmProcessFetchMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, (NodeMsgFp)vmProcessFetchMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)vmProcessWriteMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_VND_TASK_EXEC, (NodeMsgFp)vmProcessFetchMsg, 0); - - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, (NodeMsgFp)vmProcessMgmtMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE, (NodeMsgFp)vmProcessMgmtMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, (NodeMsgFp)vmProcessMgmtMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, (NodeMsgFp)vmProcessMgmtMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, (NodeMsgFp)vmProcessMgmtMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)vmProcessQueryMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)vmProcessQueryMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_TABLES_META, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONSUME, (NodeMsgFp)vmProcessQueryMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, (NodeMsgFp)vmProcessQueryMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_EXEC, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_PIPE_EXEC, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_MERGE_EXEC, (NodeMsgFp)vmProcessMergeMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, (NodeMsgFp)vmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE, (NodeMsgFp)vmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, (NodeMsgFp)vmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, (NodeMsgFp)vmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, (NodeMsgFp)vmProcessMgmtMsg, VND_VGID); } diff --git a/source/dnode/mgmt/vnode/src/vmWorker.c b/source/dnode/mgmt/vnode/src/vmWorker.c index 6c7d513c5841b33acdf83204931366a96f504ee4..4667ebc50d142160c38725a21dd9e8834202f816 100644 --- a/source/dnode/mgmt/vnode/src/vmWorker.c +++ b/source/dnode/mgmt/vnode/src/vmWorker.c @@ -117,7 +117,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO if (pRsp != NULL) { pRsp->ahandle = pRpc->ahandle; dndSendRsp(pVnode->pWrapper, pRsp); - free(pRsp); + taosMemoryFree(pRsp); } else { if (code != 0 && terrno != 0) code = terrno; vmSendRsp(pVnode->pWrapper, pMsg, code); @@ -191,6 +191,10 @@ static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EQueueTyp dTrace("msg:%p, will be written into vnode-sync queue", pMsg); code = taosWriteQitem(pVnode->pSyncQ, pMsg); break; + case MERGE_QUEUE: + dTrace("msg:%p, will be written into vnode-merge queue", pMsg); + code = taosWriteQitem(pVnode->pMergeQ, pMsg); + break; default: terrno = TSDB_CODE_INVALID_PARA; break; @@ -208,6 +212,8 @@ int32_t vmProcessQueryMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return vmPutNode int32_t vmProcessFetchMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return vmPutNodeMsgToQueue(pMgmt, pMsg, FETCH_QUEUE); } +int32_t vmProcessMergeMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return vmPutNodeMsgToQueue(pMgmt, pMsg, MERGE_QUEUE); } + int32_t vmProcessMgmtMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { SSingleWorker *pWorker = &pMgmt->mgmtWorker; dTrace("msg:%p, will be written to vnode-mgmt queue, worker:%s", pMsg, pWorker->name); @@ -239,6 +245,10 @@ static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EQueueT dTrace("msg:%p, will be put into vnode-apply queue", pMsg); code = taosWriteQitem(pVnode->pApplyQ, pMsg); break; + case MERGE_QUEUE: + dTrace("msg:%p, will be put into vnode-merge queue", pMsg); + code = taosWriteQitem(pVnode->pMergeQ, pMsg); + break; default: terrno = TSDB_CODE_INVALID_PARA; break; @@ -260,6 +270,10 @@ int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { return vmPutRpcMsgToQueue(pWrapper, pRpc, APPLY_QUEUE); } +int32_t vmPutMsgToMergeQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + return vmPutRpcMsgToQueue(pWrapper, pRpc, MERGE_QUEUE); +} + int32_t vmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype) { int32_t size = -1; SVnodeObj *pVnode = vmAcquireVnode(pWrapper->pMgmt, vgId); @@ -280,6 +294,9 @@ int32_t vmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype) { case APPLY_QUEUE: size = taosQueueSize(pVnode->pApplyQ); break; + case MERGE_QUEUE: + size = taosQueueSize(pVnode->pMergeQ); + break; default: break; } @@ -291,12 +308,13 @@ int32_t vmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype) { int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessWriteQueue); pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessApplyQueue); + pVnode->pMergeQ = tWWorkerAllocQueue(&pMgmt->mergePool, pVnode, (FItems)vmProcessMergeMsg); pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)vmProcessSyncQueue); pVnode->pFetchQ = tQWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItem)vmProcessFetchQueue); pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue); if (pVnode->pApplyQ == NULL || pVnode->pWriteQ == NULL || pVnode->pSyncQ == NULL || pVnode->pFetchQ == NULL || - pVnode->pQueryQ == NULL) { + pVnode->pQueryQ == NULL || pVnode->pMergeQ == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -310,12 +328,14 @@ void vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { tQWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ); tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ); tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ); + tWWorkerFreeQueue(&pMgmt->mergePool, pVnode->pMergeQ); tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ); pVnode->pWriteQ = NULL; pVnode->pApplyQ = NULL; pVnode->pSyncQ = NULL; pVnode->pFetchQ = NULL; pVnode->pQueryQ = NULL; + pVnode->pMergeQ = NULL; dDebug("vgId:%d, vnode queue is freed", pVnode->vgId); } @@ -326,6 +346,7 @@ int32_t vmStartWorker(SVnodesMgmt *pMgmt) { int32_t maxQueryThreads = minQueryThreads; int32_t maxWriteThreads = TMAX(tsNumOfCores, 1); int32_t maxSyncThreads = TMAX(tsNumOfCores / 2, 1); + int32_t maxMergeThreads = 1; SQWorkerPool *pQPool = &pMgmt->queryPool; pQPool->name = "vnode-query"; @@ -349,6 +370,11 @@ int32_t vmStartWorker(SVnodesMgmt *pMgmt) { pWPool->max = maxSyncThreads; if (tWWorkerInit(pWPool) != 0) return -1; + pWPool = &pMgmt->mergePool; + pWPool->name = "vnode-merge"; + pWPool->max = maxMergeThreads; + if (tWWorkerInit(pWPool) != 0) return -1; + SSingleWorkerCfg cfg = { .minNum = 1, .maxNum = 1, .name = "vnode-mgmt", .fp = (FItem)vmProcessMgmtQueue, .param = pMgmt}; if (tSingleWorkerInit(&pMgmt->mgmtWorker, &cfg) != 0) { @@ -356,7 +382,7 @@ int32_t vmStartWorker(SVnodesMgmt *pMgmt) { return -1; } - dDebug("vnode workers is initialized"); + dDebug("vnode workers are initialized"); return 0; } @@ -366,5 +392,6 @@ void vmStopWorker(SVnodesMgmt *pMgmt) { tQWorkerCleanup(&pMgmt->queryPool); tWWorkerCleanup(&pMgmt->writePool); tWWorkerCleanup(&pMgmt->syncPool); - dDebug("vnode workers is closed"); + tWWorkerCleanup(&pMgmt->mergePool); + dDebug("vnode workers are closed"); } diff --git a/source/dnode/mnode/impl/CMakeLists.txt b/source/dnode/mnode/impl/CMakeLists.txt index 514bba19f48cf1625c9d4b74c8aabb6a296ace15..6cc9fae079bbc4a754227f6c2034bf61edcfb0cb 100644 --- a/source/dnode/mnode/impl/CMakeLists.txt +++ b/source/dnode/mnode/impl/CMakeLists.txt @@ -6,7 +6,7 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( - mnode scheduler sdb wal transport cjson sync monitor + mnode scheduler sdb wal transport cjson sync monitor stream parser ) if(${BUILD_TEST}) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 909486aaac211b405a3fc468fc151f9b46573648..eb7ac5b3539ac8cbd94495583293c227671c768c 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -26,6 +26,7 @@ #include "tlog.h" #include "tmsg.h" #include "trpc.h" +#include "tstream.h" #include "ttimer.h" #include "mnode.h" @@ -103,6 +104,8 @@ typedef enum { TRN_TYPE_CREATE_STB = 4001, TRN_TYPE_ALTER_STB = 4002, TRN_TYPE_DROP_STB = 4003, + TRN_TYPE_CREATE_SMA = 4004, + TRN_TYPE_DROP_SMA = 4005, TRN_TYPE_STB_SCOPE_END, } ETrnType; @@ -265,6 +268,8 @@ typedef struct { int8_t update; int8_t cacheLastRow; int8_t streamMode; + int32_t numOfRetensions; + SArray* pRetensions; } SDbCfg; typedef struct { @@ -305,6 +310,31 @@ typedef struct { SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; } SVgObj; +typedef struct { + char name[TSDB_TABLE_FNAME_LEN]; + char stb[TSDB_TABLE_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; + int64_t createdTime; + int64_t uid; + int64_t stbUid; + int64_t dbUid; + int8_t intervalUnit; + int8_t slidingUnit; + int8_t timezone; + int32_t dstVgId; // for stream + int64_t interval; + int64_t offset; + int64_t sliding; + int32_t exprLen; // strlen + 1 + int32_t tagsFilterLen; + int32_t sqlLen; + int32_t astLen; + char* expr; + char* tagsFilter; + char* sql; + char* ast; +} SSmaObj; + typedef struct { char name[TSDB_TABLE_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN]; @@ -314,12 +344,19 @@ typedef struct { int64_t dbUid; int32_t version; int32_t nextColId; + float xFilesFactor; + int32_t aggregationMethod; + int32_t delay; + int32_t ttl; int32_t numOfColumns; int32_t numOfTags; + int32_t numOfSmas; + int32_t commentLen; SSchema* pColumns; SSchema* pTags; + SSchema* pSmas; + char* comment; SRWLatch lock; - char comment[TSDB_STB_COMMENT_LEN]; } SStbObj; typedef struct { @@ -405,7 +442,7 @@ static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsu static FORCE_INLINE void tDeleteSMqConsumerEp(SMqConsumerEp* pConsumerEp) { if (pConsumerEp) { - tfree(pConsumerEp->qmsg); + taosMemoryFreeClear(pConsumerEp->qmsg); } } @@ -474,7 +511,7 @@ typedef struct { } SMqSubscribeObj; static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() { - SMqSubscribeObj* pSub = calloc(1, sizeof(SMqSubscribeObj)); + SMqSubscribeObj* pSub = taosMemoryCalloc(1, sizeof(SMqSubscribeObj)); if (pSub == NULL) { return NULL; } @@ -501,10 +538,10 @@ static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() { return pSub; _err: - tfree(pSub->consumers); - tfree(pSub->lostConsumers); - tfree(pSub->unassignedVg); - tfree(pSub); + taosMemoryFreeClear(pSub->consumers); + taosMemoryFreeClear(pSub->lostConsumers); + taosMemoryFreeClear(pSub->unassignedVg); + taosMemoryFreeClear(pSub); return NULL; } @@ -684,6 +721,7 @@ static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pCons typedef struct { char name[TSDB_TOPIC_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN]; + char outputSTbName[TSDB_TABLE_FNAME_LEN]; int64_t createTime; int64_t updateTime; int64_t uid; @@ -692,17 +730,20 @@ typedef struct { int32_t vgNum; SRWLatch lock; int8_t status; + int8_t sourceType; + int8_t sinkType; // int32_t sqlLen; + int32_t sinkVgId; // 0 for automatic char* sql; char* logicalPlan; char* physicalPlan; - SArray* tasks; // SArray> + SArray* tasks; // SArray> + SArray* ColAlias; // SArray } SStreamObj; int32_t tEncodeSStreamObj(SCoder* pEncoder, const SStreamObj* pObj); int32_t tDecodeSStreamObj(SCoder* pDecoder, SStreamObj* pObj); - #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index d5bffada6afb6394ff6efed20c43e856d6765698..20e85973be4ec24e511ebb5fe7cb438d944b5592 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -122,9 +122,9 @@ typedef struct SMnode { SMsgCb msgCb; } SMnode; -void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp); -uint64_t mndGenerateUid(char *name, int32_t len); -void mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); +void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp); +int64_t mndGenerateUid(char *name, int32_t len); +void mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndSma.h b/source/dnode/mnode/impl/inc/mndSma.h new file mode 100644 index 0000000000000000000000000000000000000000..4a80f619d3026c82abeb2c2e9c777073dbf2fa6a --- /dev/null +++ b/source/dnode/mnode/impl/inc/mndSma.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_MND_SMA_H_ +#define _TD_MND_SMA_H_ + +#include "mndInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t mndInitSma(SMnode *pMnode); +void mndCleanupSma(SMnode *pMnode); +SSmaObj *mndAcquireSma(SMnode *pMnode, char *smaName); +void mndReleaseSma(SMnode *pMnode, SSmaObj *pSma); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_MND_SMA_H_*/ diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index b1145b0c6bf598542661f2d2f556b5e2a9f4dc60..b5d22cb7a5ba931ccbf5c5986ec41d7cbf5e6e5e 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -31,6 +31,8 @@ void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream); SSdbRaw *mndStreamActionEncode(SStreamObj *pStream); SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw); +int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index a6e6d57345ceca265575bb075b58825489eb4d14..68c8c6d12744b79433b5f00a3bfe3f97dab0495b 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -158,7 +158,7 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) { ACCT_DECODE_OVER: if (terrno != 0) { mError("acct:%s, failed to decode from raw:%p since %s", pAcct->acct, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index d9a0df555c33287eeae890c538d17cc6da2ff8ed..c4868d980230c64ddbdfe05fff78566767f0d9fc 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -126,7 +126,7 @@ static SSdbRow *mndBnodeActionDecode(SSdbRaw *pRaw) { BNODE_DECODE_OVER: if (terrno != 0) { mError("bnode:%d, failed to decode from raw:%p since %s", pObj->id, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -191,7 +191,7 @@ static int32_t mndSetCreateBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S createReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -206,7 +206,7 @@ static int32_t mndSetCreateBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } @@ -218,7 +218,7 @@ static int32_t mndSetCreateBnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S dropReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -233,7 +233,7 @@ static int32_t mndSetCreateBnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } @@ -341,7 +341,7 @@ static int32_t mndSetDropBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBn dropReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -356,7 +356,7 @@ static int32_t mndSetDropBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBn action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index ce470806452b4ac50c7e6f8e9e1d78755619c466..dde7e1fe8fd946c0f9632dcb7273b89cbe246757 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -116,7 +116,7 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) { CLUSTER_DECODE_OVER: if (terrno != 0) { mError("cluster:%" PRId64 ", failed to decode from raw:%p since %s", pCluster->id, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index d86665b832e63ed59943b3500c723f036bb55d60..8eceea3d06b43b66bf3ef0ba482c16e6d008170f 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -54,7 +54,7 @@ int32_t mndInitConsumer(SMnode *pMnode) { void mndCleanupConsumer(SMnode *pMnode) {} SMqConsumerObj *mndCreateConsumer(int64_t consumerId, const char *cgroup) { - SMqConsumerObj *pConsumer = calloc(1, sizeof(SMqConsumerObj)); + SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj)); if (pConsumer == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -79,7 +79,7 @@ SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { SSdbRaw *pRaw = sdbAllocRaw(SDB_CONSUMER, MND_CONSUMER_VER_NUMBER, size); if (pRaw == NULL) goto CM_ENCODE_OVER; - buf = malloc(tlen); + buf = taosMemoryMalloc(tlen); if (buf == NULL) goto CM_ENCODE_OVER; void *abuf = buf; @@ -94,7 +94,7 @@ SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { terrno = TSDB_CODE_SUCCESS; CM_ENCODE_OVER: - tfree(buf); + taosMemoryFreeClear(buf); if (terrno != 0) { mError("consumer:%" PRId64 ", failed to encode to raw:%p since %s", pConsumer->consumerId, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -126,7 +126,7 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; int32_t len; SDB_GET_INT32(pRaw, dataPos, &len, CM_DECODE_OVER); - buf = malloc(len); + buf = taosMemoryMalloc(len); if (buf == NULL) goto CM_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, buf, len, CM_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CM_DECODE_OVER); @@ -138,10 +138,10 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_SUCCESS; CM_DECODE_OVER: - tfree(buf); + taosMemoryFreeClear(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("consumer:%" PRId64 ", failed to decode from raw:%p since %s", pConsumer->consumerId, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 7469d720b5bc030192e090f5926406dc0071017a..c556cfc71e8d5ab27ec9cf28aebf6d0dd5e928bf 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -100,6 +100,15 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { SDB_SET_INT8(pRaw, dataPos, pDb->cfg.quorum, DB_ENCODE_OVER) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.update, DB_ENCODE_OVER) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.cacheLastRow, DB_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfRetensions, DB_ENCODE_OVER) + for (int32_t i = 0; i < pDb->cfg.numOfRetensions; ++i) { + SRetention *pRetension = taosArrayGet(pDb->cfg.pRetensions, i); + SDB_SET_INT32(pRaw, dataPos, pRetension->freq, DB_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pRetension->keep, DB_ENCODE_OVER) + SDB_SET_INT8(pRaw, dataPos, pRetension->freqUnit, DB_ENCODE_OVER) + SDB_SET_INT8(pRaw, dataPos, pRetension->keepUnit, DB_ENCODE_OVER) + } + SDB_SET_RESERVE(pRaw, dataPos, TSDB_DB_RESERVE_SIZE, DB_ENCODE_OVER) SDB_SET_DATALEN(pRaw, dataPos, DB_ENCODE_OVER) @@ -161,6 +170,22 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.quorum, DB_DECODE_OVER) SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.update, DB_DECODE_OVER) SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.cacheLastRow, DB_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.numOfRetensions, DB_DECODE_OVER) + if (pDb->cfg.numOfRetensions > 0) { + pDb->cfg.pRetensions = taosArrayInit(pDb->cfg.numOfRetensions, sizeof(SRetention)); + if (pDb->cfg.pRetensions == NULL) goto DB_DECODE_OVER; + for (int32_t i = 0; i < pDb->cfg.numOfRetensions; ++i) { + SRetention retension = {0}; + SDB_GET_INT32(pRaw, dataPos, &retension.freq, DB_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &retension.keep, DB_DECODE_OVER) + SDB_GET_INT8(pRaw, dataPos, &retension.freqUnit, DB_DECODE_OVER) + SDB_GET_INT8(pRaw, dataPos, &retension.keepUnit, DB_DECODE_OVER) + if (taosArrayPush(pDb->cfg.pRetensions, &retension) == NULL) { + goto DB_DECODE_OVER; + } + } + } + SDB_GET_RESERVE(pRaw, dataPos, TSDB_DB_RESERVE_SIZE, DB_DECODE_OVER) terrno = 0; @@ -168,7 +193,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { DB_DECODE_OVER: if (terrno != 0) { mError("db:%s, failed to decode from raw:%p since %s", pDb->name, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -183,6 +208,7 @@ static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb) { static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb) { mTrace("db:%s, perform delete action, row:%p", pDb->name, pDb); + taosArrayDestroy(pDb->cfg.pRetensions); return 0; } @@ -344,7 +370,7 @@ static int32_t mndSetCreateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.msgType = TDMT_DND_CREATE_VNODE; action.acceptableCode = TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } } @@ -375,7 +401,7 @@ static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.msgType = TDMT_DND_DROP_VNODE; action.acceptableCode = TSDB_CODE_DND_VNODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } } @@ -417,6 +443,10 @@ static int32_t mndCreateDb(SMnode *pMnode, SNodeMsg *pReq, SCreateDbReq *pCreate .streamMode = pCreate->streamMode, }; + dbObj.cfg.numOfRetensions = pCreate->numOfRetensions; + dbObj.cfg.pRetensions = pCreate->pRetensions; + pCreate = NULL; + mndSetDefaultDbCfg(&dbObj.cfg); if (mndCheckDbName(dbObj.name, pUser) != 0) { @@ -453,7 +483,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SNodeMsg *pReq, SCreateDbReq *pCreate code = 0; CREATE_DB_OVER: - free(pVgroups); + taosMemoryFree(pVgroups); mndTransDrop(pTrans); return code; } @@ -505,6 +535,7 @@ CREATE_DB_OVER: mndReleaseDb(pMnode, pDb); mndReleaseUser(pMnode, pUser); + tFreeSCreateDbReq(&createReq); return code; } @@ -591,7 +622,7 @@ static int32_t mndBuildUpdateVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj action.contLen = contLen; action.msgType = TDMT_DND_ALTER_VNODE; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } } @@ -771,7 +802,7 @@ static int32_t mndBuildDropVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj * action.msgType = TDMT_DND_DROP_VNODE; action.acceptableCode = TSDB_CODE_DND_VNODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } } @@ -814,7 +845,7 @@ static int32_t mndBuildDropDbRsp(SDbObj *pDb, int32_t *pRspLen, void **ppRsp, bo if (useRpcMalloc) { pRsp = rpcMallocCont(rspLen); } else { - pRsp = malloc(rspLen); + pRsp = taosMemoryMalloc(rspLen); } if (pRsp == NULL) { @@ -1125,7 +1156,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, } int32_t rspLen = tSerializeSUseDbBatchRsp(NULL, 0, &batchUseRsp); - void *pRsp = malloc(rspLen); + void *pRsp = taosMemoryMalloc(rspLen); if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tFreeSUseDbBatchRsp(&batchUseRsp); diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index f81dead3259c3e43d0287829dfcd33be6ba2fad1..6fa926d548a1955cdb396b914101c1a5b7cbb113 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -16,6 +16,8 @@ #include "mndDef.h" int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) { + int32_t sz = 0; + int32_t outputNameSz = 0; if (tEncodeCStr(pEncoder, pObj->name) < 0) return -1; if (tEncodeCStr(pEncoder, pObj->db) < 0) return -1; if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1; @@ -29,19 +31,27 @@ int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) { if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1; // TODO encode tasks if (pObj->tasks) { - int32_t sz = taosArrayGetSize(pObj->tasks); - tEncodeI32(pEncoder, sz); - for (int32_t i = 0; i < sz; i++) { - SArray *pArray = taosArrayGet(pObj->tasks, i); - int32_t innerSz = taosArrayGetSize(pArray); - tEncodeI32(pEncoder, innerSz); - for (int32_t j = 0; j < innerSz; j++) { - SStreamTask *pTask = taosArrayGet(pArray, j); - tEncodeSStreamTask(pEncoder, pTask); - } + sz = taosArrayGetSize(pObj->tasks); + } + if (tEncodeI32(pEncoder, sz) < 0) return -1; + + for (int32_t i = 0; i < sz; i++) { + SArray *pArray = taosArrayGet(pObj->tasks, i); + int32_t innerSz = taosArrayGetSize(pArray); + if (tEncodeI32(pEncoder, innerSz) < 0) return -1; + for (int32_t j = 0; j < innerSz; j++) { + SStreamTask *pTask = taosArrayGet(pArray, j); + if (tEncodeSStreamTask(pEncoder, pTask) < 0) return -1; } - } else { - tEncodeI32(pEncoder, 0); + } + + if (pObj->ColAlias != NULL) { + outputNameSz = taosArrayGetSize(pObj->ColAlias); + } + if (tEncodeI32(pEncoder, outputNameSz) < 0) return -1; + for (int32_t i = 0; i < outputNameSz; i++) { + char *name = taosArrayGetP(pObj->ColAlias, i); + if (tEncodeCStr(pEncoder, name) < 0) return -1; } return pEncoder->pos; } @@ -58,6 +68,7 @@ int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) { if (tDecodeCStrAlloc(pDecoder, &pObj->sql) < 0) return -1; if (tDecodeCStrAlloc(pDecoder, &pObj->logicalPlan) < 0) return -1; if (tDecodeCStrAlloc(pDecoder, &pObj->physicalPlan) < 0) return -1; + pObj->tasks = NULL; int32_t sz; if (tDecodeI32(pDecoder, &sz) < 0) return -1; if (sz != 0) { @@ -73,8 +84,19 @@ int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) { } taosArrayPush(pObj->tasks, pArray); } - } else { - pObj->tasks = NULL; + } + int32_t outputNameSz; + if (tDecodeI32(pDecoder, &outputNameSz) < 0) return -1; + if (outputNameSz != 0) { + pObj->ColAlias = taosArrayInit(outputNameSz, sizeof(void *)); + if (pObj->ColAlias == NULL) { + return -1; + } + } + for (int32_t i = 0; i < outputNameSz; i++) { + char *name; + if (tDecodeCStrAlloc(pDecoder, &name) < 0) return -1; + taosArrayPush(pObj->ColAlias, &name); } return 0; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 3e47f3a9e85eef453af892afe2c9b7f7cc1f4ef3..55414124606b6f526300d94d2575e14ecbb5332f 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -164,7 +164,7 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) { DNODE_DECODE_OVER: if (terrno != 0) { mError("dnode:%d, failed to decode from raw:%p since %s", pDnode->id, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index b43d887dd79f9ad164d40722c8677474e155dcac..be4198f0d62c64896e28ac52e3b52c04e71ca031 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -127,8 +127,8 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pFunc->commentSize, FUNC_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pFunc->codeSize, FUNC_DECODE_OVER) - pFunc->pComment = calloc(1, pFunc->commentSize); - pFunc->pCode = calloc(1, pFunc->codeSize); + pFunc->pComment = taosMemoryCalloc(1, pFunc->commentSize); + pFunc->pCode = taosMemoryCalloc(1, pFunc->codeSize); if (pFunc->pComment == NULL || pFunc->pCode == NULL) { goto FUNC_DECODE_OVER; } @@ -142,7 +142,7 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) { FUNC_DECODE_OVER: if (terrno != 0) { mError("func:%s, failed to decode from raw:%p since %s", pFunc->name, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -157,8 +157,8 @@ static int32_t mndFuncActionInsert(SSdb *pSdb, SFuncObj *pFunc) { static int32_t mndFuncActionDelete(SSdb *pSdb, SFuncObj *pFunc) { mTrace("func:%s, perform delete action, row:%p", pFunc->name, pFunc); - tfree(pFunc->pCode); - tfree(pFunc->pComment); + taosMemoryFreeClear(pFunc->pCode); + taosMemoryFreeClear(pFunc->pComment); return 0; } @@ -196,8 +196,8 @@ static int32_t mndCreateFunc(SMnode *pMnode, SNodeMsg *pReq, SCreateFuncReq *pCr func.signature = pCreate->signature; func.commentSize = pCreate->commentSize; func.codeSize = pCreate->codeSize; - func.pComment = malloc(func.commentSize); - func.pCode = malloc(func.codeSize); + func.pComment = taosMemoryMalloc(func.commentSize); + func.pCode = taosMemoryMalloc(func.codeSize); if (func.pCode == NULL || func.pCode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto CREATE_FUNC_OVER; @@ -228,8 +228,8 @@ static int32_t mndCreateFunc(SMnode *pMnode, SNodeMsg *pReq, SCreateFuncReq *pCr code = 0; CREATE_FUNC_OVER: - free(func.pCode); - free(func.pComment); + taosMemoryFree(func.pCode); + taosMemoryFree(func.pComment); mndTransDrop(pTrans); return code; } diff --git a/source/dnode/mnode/impl/src/mndInfoSchema.c b/source/dnode/mnode/impl/src/mndInfoSchema.c index 964c4ab424e4f6de0015413429e5b0c7d668e555..5f369e7088dba40e03fd6d95ef0cece721739e38 100644 --- a/source/dnode/mnode/impl/src/mndInfoSchema.c +++ b/source/dnode/mnode/impl/src/mndInfoSchema.c @@ -152,7 +152,7 @@ static const SInfosTableMeta infosMeta[] = {{TSDB_INS_TABLE_DNODES, dnodesSchema //connection/application/ int32_t mndInitInfosTableSchema(const SInfosTableSchema *pSrc, int32_t colNum, SSchema **pDst) { - SSchema *schema = calloc(colNum, sizeof(SSchema)); + SSchema *schema = taosMemoryCalloc(colNum, sizeof(SSchema)); if (NULL == schema) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -210,7 +210,7 @@ int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char * *pRsp = *meta; - pRsp->pSchemas = calloc(meta->numOfColumns, sizeof(SSchema)); + pRsp->pSchemas = taosMemoryCalloc(meta->numOfColumns, sizeof(SSchema)); if (pRsp->pSchemas == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; pRsp->pSchemas = NULL; @@ -241,7 +241,7 @@ void mndCleanupInfos(SMnode *pMnode) { while (pIter) { STableMetaRsp *meta = (STableMetaRsp *)pIter; - tfree(meta->pSchemas); + taosMemoryFreeClear(meta->pSchemas); pIter = taosHashIterate(pMnode->infosMeta, pIter); } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 703f36bb335248a8ff4db645210643529b6553cc..c408fea636b03b87e915995ad01a66981eb2a1cb 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -180,7 +180,7 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) { MNODE_DECODE_OVER: if (terrno != 0) { mError("mnode:%d, failed to decode from raw:%p since %s", pObj->id, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -313,7 +313,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno createReq.dnodeId = pMObj->id; int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &createReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); tSerializeSDCreateMnodeReq(pReq, contLen, &createReq); action.epSet = mndGetDnodeEpset(pMObj->pDnode); @@ -323,7 +323,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pMObj); return -1; @@ -338,7 +338,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno createReq.dnodeId = pObj->id; int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &createReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); tSerializeSDCreateMnodeReq(pReq, contLen, &createReq); action.epSet = mndGetDnodeEpset(pDnode); @@ -347,7 +347,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno action.msgType = TDMT_DND_CREATE_MNODE; action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } } @@ -483,7 +483,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode alterReq.dnodeId = pMObj->id; int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &alterReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq); action.epSet = mndGetDnodeEpset(pMObj->pDnode); @@ -493,7 +493,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pMObj); return -1; @@ -510,7 +510,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode SDDropMnodeReq dropReq = {0}; dropReq.dnodeId = pObj->id; int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &dropReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); tSerializeSMCreateDropMnodeReq(pReq, contLen, &dropReq); action.epSet = mndGetDnodeEpset(pDnode); @@ -519,7 +519,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode action.msgType = TDMT_DND_DROP_MNODE; action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } } diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c index 38157cf220b6dfccb740fa1264c343f9625eb6a8..22dad48772b2024cb2bb96243de070300aec4e0a 100644 --- a/source/dnode/mnode/impl/src/mndOffset.c +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -59,7 +59,7 @@ SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset) { SSdbRaw *pRaw = sdbAllocRaw(SDB_OFFSET, MND_OFFSET_VER_NUMBER, size); if (pRaw == NULL) goto OFFSET_ENCODE_OVER; - buf = malloc(tlen); + buf = taosMemoryMalloc(tlen); if (buf == NULL) goto OFFSET_ENCODE_OVER; void *abuf = buf; @@ -74,7 +74,7 @@ SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset) { terrno = TSDB_CODE_SUCCESS; OFFSET_ENCODE_OVER: - tfree(buf); + taosMemoryFreeClear(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("offset:%s, failed to encode to raw:%p since %s", pOffset->key, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -107,7 +107,7 @@ SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; int32_t tlen; SDB_GET_INT32(pRaw, dataPos, &tlen, OFFSET_DECODE_OVER); - buf = malloc(tlen + 1); + buf = taosMemoryMalloc(tlen + 1); if (buf == NULL) goto OFFSET_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OFFSET_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_OFFSET_RESERVE_SIZE, OFFSET_DECODE_OVER); @@ -119,10 +119,10 @@ SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_SUCCESS; OFFSET_DECODE_OVER: - tfree(buf); + taosMemoryFreeClear(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("offset:%s, failed to decode from raw:%p since %s", pOffset->key, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 7ca1984457daef36d1b7bf4a3dee081a67d65a9e..bf378a4d43f02c28479f194f02e457e8c0c5c040 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -130,7 +130,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid, } static void mndFreeConn(SConnObj *pConn) { - tfree(pConn->pQueries); + taosMemoryFreeClear(pConn->pQueries); mTrace("conn:%d, is destroyed, data:%p", pConn->id, pConn); } @@ -260,7 +260,7 @@ static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatReq *pReq) { if (numOfQueries > 0) { if (pConn->pQueries == NULL) { - pConn->pQueries = calloc(sizeof(SQueryDesc), QUERY_SAVE_SIZE); + pConn->pQueries = taosMemoryCalloc(sizeof(SQueryDesc), QUERY_SAVE_SIZE); } pConn->numOfQueries = TMIN(QUERY_SAVE_SIZE, numOfQueries); @@ -276,7 +276,7 @@ static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatReq *pReq) { static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) { #if 0 - SClientHbRsp* pRsp = malloc(sizeof(SClientHbRsp)); + SClientHbRsp* pRsp = taosMemoryMalloc(sizeof(SClientHbRsp)); if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -292,7 +292,7 @@ static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) { SHashObj* pObj = pReq->info; SKv* pKv = taosHashGet(pObj, "mq-tmp", strlen("mq-tmp") + 1); if (pKv == NULL) { - free(pRsp); + taosMemoryFree(pRsp); return NULL; } SMqHbMsg mqHb; @@ -325,7 +325,7 @@ static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) { taosArrayPush(batchRsp.batchRsps, &innerBatchRsp); } int32_t tlen = taosEncodeSMqHbBatchRsp(NULL, &batchRsp); - void* buf = malloc(tlen); + void* buf = taosMemoryMalloc(tlen); if (buf == NULL) { //TODO return NULL; @@ -402,7 +402,7 @@ static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq) { SClientHbRsp *pRsp = mndMqHbBuildRsp(pMnode, pHbReq); if (pRsp != NULL) { taosArrayPush(batchRsp.rsps, pRsp); - free(pRsp); + taosMemoryFree(pRsp); } } } @@ -418,7 +418,7 @@ static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq) { int32_t kvNum = (rsp->info) ? taosArrayGetSize(rsp->info) : 0; for (int32_t n = 0; n < kvNum; ++n) { SKv *kv = taosArrayGet(rsp->info, n); - tfree(kv->value); + taosMemoryFreeClear(kv->value); } taosArrayDestroy(rsp->info); } diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 3c66160c916735515caa411b0712951db34f12e2..4b19a26bc4f2dc7a6c039b551b17835d347951f0 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -128,7 +128,7 @@ static SSdbRow *mndQnodeActionDecode(SSdbRaw *pRaw) { QNODE_DECODE_OVER: if (terrno != 0) { mError("qnode:%d, failed to decode from raw:%p since %s", pObj->id, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -193,7 +193,7 @@ static int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S createReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -208,7 +208,7 @@ static int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } @@ -220,7 +220,7 @@ static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S dropReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -235,7 +235,7 @@ static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } @@ -343,7 +343,7 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn dropReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -358,7 +358,7 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } @@ -474,7 +474,7 @@ static int32_t mndProcessQnodeListReq(SNodeMsg *pReq) { } int32_t rspLen = tSerializeSQnodeListRsp(NULL, 0, &qlistRsp); - void *pRsp = malloc(rspLen); + void *pRsp = taosMemoryMalloc(rspLen); if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto QNODE_LIST_OVER; diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index b95574ea4173837bfd0ad9e68b489255c6ec425d..a59aa3c7e597617334b372dcaca7aeb5a9ad91ae 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -32,20 +32,23 @@ #include "tname.h" #include "tuuid.h" -int32_t mndPersistTaskDeployReq(STrans* pTrans, SStreamTask* pTask, const SEpSet* pEpSet, tmsg_t type) { +extern bool tsStreamSchedV; + +int32_t mndPersistTaskDeployReq(STrans* pTrans, SStreamTask* pTask, const SEpSet* pEpSet, tmsg_t type, int32_t nodeId) { SCoder encoder; tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); tEncodeSStreamTask(&encoder, pTask); - int32_t tlen = sizeof(SMsgHead) + encoder.pos; + int32_t size = encoder.pos; + int32_t tlen = sizeof(SMsgHead) + size; tCoderClear(&encoder); - void* buf = malloc(tlen); + void* buf = taosMemoryMalloc(tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - ((SMsgHead*)buf)->streamTaskId = pTask->taskId; + ((SMsgHead*)buf)->vgId = htonl(nodeId); void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tCoderInit(&encoder, TD_LITTLE_ENDIAN, abuf, tlen, TD_ENCODER); + tCoderInit(&encoder, TD_LITTLE_ENDIAN, abuf, size, TD_ENCODER); tEncodeSStreamTask(&encoder, pTask); tCoderClear(&encoder); @@ -55,7 +58,7 @@ int32_t mndPersistTaskDeployReq(STrans* pTrans, SStreamTask* pTask, const SEpSet action.contLen = tlen; action.msgType = type; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - rpcFreeCont(buf); + taosMemoryFree(buf); return -1; } return 0; @@ -66,11 +69,11 @@ int32_t mndAssignTaskToVg(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, SS plan->execNode.nodeId = pVgroup->vgId; plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup); - if (qSubPlanToString(plan, &pTask->qmsg, &msgLen) < 0) { + if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) { terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } - mndPersistTaskDeployReq(pTrans, pTask, &plan->execNode.epSet, TDMT_VND_TASK_DEPLOY); + mndPersistTaskDeployReq(pTrans, pTask, &plan->execNode.epSet, TDMT_VND_TASK_DEPLOY, pVgroup->vgId); return 0; } @@ -86,11 +89,11 @@ int32_t mndAssignTaskToSnode(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, plan->execNode.nodeId = pSnode->id; plan->execNode.epSet = mndAcquireEpFromSnode(pMnode, pSnode); - if (qSubPlanToString(plan, &pTask->qmsg, &msgLen) < 0) { + if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) { terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } - mndPersistTaskDeployReq(pTrans, pTask, &plan->execNode.epSet, TDMT_SND_TASK_DEPLOY); + mndPersistTaskDeployReq(pTrans, pTask, &plan->execNode.epSet, TDMT_SND_TASK_DEPLOY, 0); return 0; } @@ -106,16 +109,23 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { int32_t totLevel = LIST_LENGTH(pPlan->pSubplans); pStream->tasks = taosArrayInit(totLevel, sizeof(SArray)); + int32_t lastUsedVgId = 0; + + // gather vnodes + // gather snodes + // iterate plan, expand source to vnodes and assign ep to each task + // iterate tasks, assign sink type and sink ep to each task - for (int32_t level = 0; level < totLevel; level++) { + for (int32_t revLevel = totLevel - 1; revLevel >= 0; revLevel--) { + int32_t level = totLevel - 1 - revLevel; SArray* taskOneLevel = taosArrayInit(0, sizeof(SStreamTask)); - SNodeListNode* inner = nodesListGetNode(pPlan->pSubplans, level); + SNodeListNode* inner = nodesListGetNode(pPlan->pSubplans, revLevel); int32_t opNum = LIST_LENGTH(inner->pNodeList); ASSERT(opNum == 1); - SSubplan* plan = nodesListGetNode(inner->pNodeList, level); + SSubplan* plan = nodesListGetNode(inner->pNodeList, 0); if (level == 0) { - ASSERT(plan->type == SUBPLAN_TYPE_SCAN); + ASSERT(plan->subplanType == SUBPLAN_TYPE_SCAN); void* pIter = NULL; while (1) { pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); @@ -125,12 +135,15 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { continue; } + lastUsedVgId = pVgroup->vgId; pStream->vgNum++; - // send to vnode - SStreamTask* pTask = streamTaskNew(pStream->uid, level); - pTask->pipeSink = level == totLevel - 1 ? 1 : 0; - // TODO: set to + SStreamTask* pTask = tNewSStreamTask(pStream->uid); + /*pTask->level = level;*/ + // TODO + /*pTask->sourceType = STREAM_SOURCE__SUPER;*/ + /*pTask->sinkType = level == totLevel - 1 ? 1 : 0;*/ + pTask->exec.parallelizable = 1; if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); @@ -139,24 +152,36 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { taosArrayPush(taskOneLevel, pTask); } } else { - SStreamTask* pTask = streamTaskNew(pStream->uid, level); - pTask->pipeSink = level == totLevel - 1 ? 1 : 0; + SStreamTask* pTask = tNewSStreamTask(pStream->uid); + /*pTask->level = level;*/ + /*pTask->sourceType = STREAM_SOURCE__NONE;*/ + /*pTask->sinkType = level == totLevel - 1 ? 1 : 0;*/ + pTask->exec.parallelizable = plan->subplanType == SUBPLAN_TYPE_SCAN; + SSnodeObj* pSnode = mndSchedFetchSnode(pMnode); - if (pSnode != NULL) { + if (pSnode == NULL || tsStreamSchedV) { + ASSERT(lastUsedVgId != 0); + SVgObj* pVg = mndAcquireVgroup(pMnode, lastUsedVgId); + if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVg) < 0) { + sdbRelease(pSdb, pVg); + qDestroyQueryPlan(pPlan); + return -1; + } + sdbRelease(pSdb, pVg); + } else { if (mndAssignTaskToSnode(pMnode, pTrans, pTask, plan, pSnode) < 0) { sdbRelease(pSdb, pSnode); qDestroyQueryPlan(pPlan); return -1; } - sdbRelease(pMnode->pSdb, pSnode); - } else { - // TODO: assign to one vg - ASSERT(0); } + sdbRelease(pMnode->pSdb, pSnode); + taosArrayPush(taskOneLevel, pTask); } taosArrayPush(pStream->tasks, taskOneLevel); } + qDestroyQueryPlan(pPlan); return 0; } diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 8536ee89814972532a28737c4da0490a23c890c2..a830ee7104c2ff6802d3e718a818920c87a114fa 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -146,7 +146,7 @@ static int32_t mndProcessShowReq(SNodeMsg *pReq) { } showRsp.showId = pShow->id; - showRsp.tableMeta.pSchemas = calloc(TSDB_MAX_COLUMNS, sizeof(SSchema)); + showRsp.tableMeta.pSchemas = taosMemoryCalloc(TSDB_MAX_COLUMNS, sizeof(SSchema)); if (showRsp.tableMeta.pSchemas == NULL) { mndReleaseShowObj(pShow, true); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -427,6 +427,8 @@ char *mndShowStr(int32_t showType) { return "show topics"; case TSDB_MGMT_TABLE_FUNC: return "show functions"; + case TSDB_MGMT_TABLE_INDEX: + return "show indexes"; default: return "undefined"; } diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c new file mode 100644 index 0000000000000000000000000000000000000000..146975aa38851253bc1ca8f656de672a11dc666f --- /dev/null +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -0,0 +1,784 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndSma.h" +#include "mndAuth.h" +#include "mndDb.h" +#include "mndDnode.h" +#include "mndInfoSchema.h" +#include "mndMnode.h" +#include "mndShow.h" +#include "mndStb.c" +#include "mndStream.h" +#include "mndTrans.h" +#include "mndUser.h" +#include "mndVgroup.h" +#include "tname.h" + +#define TSDB_SMA_VER_NUMBER 1 +#define TSDB_SMA_RESERVE_SIZE 64 + +static SSdbRaw *mndSmaActionEncode(SSmaObj *pSma); +static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw); +static int32_t mndSmaActionInsert(SSdb *pSdb, SSmaObj *pSma); +static int32_t mndSmaActionDelete(SSdb *pSdb, SSmaObj *pSpSmatb); +static int32_t mndSmaActionUpdate(SSdb *pSdb, SSmaObj *pOld, SSmaObj *pNew); +static int32_t mndProcessMCreateSmaReq(SNodeMsg *pReq); +static int32_t mndProcessMDropSmaReq(SNodeMsg *pReq); +static int32_t mndProcessVCreateSmaRsp(SNodeMsg *pRsp); +static int32_t mndProcessVDropSmaRsp(SNodeMsg *pRsp); +static int32_t mndGetSmaMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveSma(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextSma(SMnode *pMnode, void *pIter); + +int32_t mndInitSma(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_SMA, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndSmaActionEncode, + .decodeFp = (SdbDecodeFp)mndSmaActionDecode, + .insertFp = (SdbInsertFp)mndSmaActionInsert, + .updateFp = (SdbUpdateFp)mndSmaActionUpdate, + .deleteFp = (SdbDeleteFp)mndSmaActionDelete}; + + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_SMA, mndProcessMCreateSmaReq); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_SMA, mndProcessMDropSmaReq); + mndSetMsgHandle(pMnode, TDMT_VND_CREATE_SMA_RSP, mndProcessVCreateSmaRsp); + mndSetMsgHandle(pMnode, TDMT_VND_DROP_SMA_RSP, mndProcessVDropSmaRsp); + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndGetSmaMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndRetrieveSma); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndCancelGetNextSma); + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupSma(SMnode *pMnode) {} + +static SSdbRaw *mndSmaActionEncode(SSmaObj *pSma) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + + int32_t size = sizeof(SSmaObj) + pSma->exprLen + pSma->tagsFilterLen + pSma->sqlLen + pSma->astLen + TSDB_SMA_RESERVE_SIZE; + SSdbRaw *pRaw = sdbAllocRaw(SDB_SMA, TSDB_SMA_VER_NUMBER, size); + if (pRaw == NULL) goto _OVER; + + int32_t dataPos = 0; + + SDB_SET_BINARY(pRaw, dataPos, pSma->name, TSDB_TABLE_FNAME_LEN, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pSma->stb, TSDB_TABLE_FNAME_LEN, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pSma->db, TSDB_DB_FNAME_LEN, _OVER) + SDB_SET_INT64(pRaw, dataPos, pSma->createdTime, _OVER) + SDB_SET_INT64(pRaw, dataPos, pSma->uid, _OVER) + SDB_SET_INT64(pRaw, dataPos, pSma->stbUid, _OVER) + SDB_SET_INT64(pRaw, dataPos, pSma->dbUid, _OVER) + SDB_SET_INT8(pRaw, dataPos, pSma->intervalUnit, _OVER) + SDB_SET_INT8(pRaw, dataPos, pSma->slidingUnit, _OVER) + SDB_SET_INT8(pRaw, dataPos, pSma->timezone, _OVER) + SDB_SET_INT32(pRaw, dataPos, pSma->dstVgId, _OVER) + SDB_SET_INT64(pRaw, dataPos, pSma->interval, _OVER) + SDB_SET_INT64(pRaw, dataPos, pSma->offset, _OVER) + SDB_SET_INT64(pRaw, dataPos, pSma->sliding, _OVER) + SDB_SET_INT32(pRaw, dataPos, pSma->exprLen, _OVER) + SDB_SET_INT32(pRaw, dataPos, pSma->tagsFilterLen, _OVER) + SDB_SET_INT32(pRaw, dataPos, pSma->sqlLen, _OVER) + SDB_SET_INT32(pRaw, dataPos, pSma->astLen, _OVER) + if (pSma->exprLen > 0) { + SDB_SET_BINARY(pRaw, dataPos, pSma->expr, pSma->exprLen, _OVER) + } + if (pSma->tagsFilterLen > 0) { + SDB_SET_BINARY(pRaw, dataPos, pSma->tagsFilter, pSma->tagsFilterLen, _OVER) + } + if (pSma->sqlLen > 0) { + SDB_SET_BINARY(pRaw, dataPos, pSma->sql, pSma->sqlLen, _OVER) + } + if (pSma->astLen > 0) { + SDB_SET_BINARY(pRaw, dataPos, pSma->ast, pSma->astLen, _OVER) + } + + SDB_SET_RESERVE(pRaw, dataPos, TSDB_SMA_RESERVE_SIZE, _OVER) + SDB_SET_DATALEN(pRaw, dataPos, _OVER) + terrno = 0; + +_OVER: + if (terrno != 0) { + mError("sma:%s, failed to encode to raw:%p since %s", pSma->name, pRaw, terrstr()); + sdbFreeRaw(pRaw); + return NULL; + } + + mTrace("sma:%s, encode to raw:%p, row:%p", pSma->name, pRaw, pSma); + return pRaw; +} + +static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; + + if (sver != TSDB_SMA_VER_NUMBER) { + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + goto _OVER; + } + + SSdbRow *pRow = sdbAllocRow(sizeof(SSmaObj)); + if (pRow == NULL) goto _OVER; + + SSmaObj *pSma = sdbGetRowObj(pRow); + if (pSma == NULL) goto _OVER; + + int32_t dataPos = 0; + + SDB_GET_BINARY(pRaw, dataPos, pSma->name, TSDB_TABLE_FNAME_LEN, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pSma->stb, TSDB_TABLE_FNAME_LEN, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pSma->db, TSDB_DB_FNAME_LEN, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pSma->createdTime, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pSma->uid, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pSma->stbUid, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pSma->dbUid, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pSma->intervalUnit, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pSma->slidingUnit, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pSma->timezone, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pSma->dstVgId, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pSma->interval, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pSma->offset, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pSma->sliding, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pSma->exprLen, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pSma->tagsFilterLen, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pSma->sqlLen, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pSma->astLen, _OVER) + + if (pSma->exprLen > 0) { + pSma->expr = taosMemoryCalloc(pSma->exprLen, 1); + if (pSma->expr == NULL) goto _OVER; + SDB_GET_BINARY(pRaw, dataPos, pSma->expr, pSma->exprLen, _OVER) + } + + if (pSma->tagsFilterLen > 0) { + pSma->tagsFilter = taosMemoryCalloc(pSma->tagsFilterLen, 1); + if (pSma->tagsFilter == NULL) goto _OVER; + SDB_GET_BINARY(pRaw, dataPos, pSma->tagsFilter, pSma->tagsFilterLen, _OVER) + } + + if (pSma->sqlLen > 0) { + pSma->sql = taosMemoryCalloc(pSma->sqlLen, 1); + if (pSma->sql == NULL) goto _OVER; + SDB_GET_BINARY(pRaw, dataPos, pSma->sql, pSma->sqlLen, _OVER) + } + + if (pSma->astLen > 0) { + pSma->ast = taosMemoryCalloc(pSma->astLen, 1); + if (pSma->ast == NULL) goto _OVER; + SDB_GET_BINARY(pRaw, dataPos, pSma->ast, pSma->astLen, _OVER) + } + + SDB_GET_RESERVE(pRaw, dataPos, TSDB_SMA_RESERVE_SIZE, _OVER) + terrno = 0; + +_OVER: + if (terrno != 0) { + mError("sma:%s, failed to decode from raw:%p since %s", pSma->name, pRaw, terrstr()); + taosMemoryFreeClear(pSma->expr); + taosMemoryFreeClear(pSma->tagsFilter); + taosMemoryFreeClear(pRow); + return NULL; + } + + mTrace("sma:%s, decode from raw:%p, row:%p", pSma->name, pRaw, pSma); + return pRow; +} + +static int32_t mndSmaActionInsert(SSdb *pSdb, SSmaObj *pSma) { + mTrace("sma:%s, perform insert action, row:%p", pSma->name, pSma); + return 0; +} + +static int32_t mndSmaActionDelete(SSdb *pSdb, SSmaObj *pSma) { + mTrace("sma:%s, perform delete action, row:%p", pSma->name, pSma); + taosMemoryFreeClear(pSma->tagsFilter); + taosMemoryFreeClear(pSma->expr); + return 0; +} + +static int32_t mndSmaActionUpdate(SSdb *pSdb, SSmaObj *pOld, SSmaObj *pNew) { + mTrace("sma:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); + return 0; +} + +SSmaObj *mndAcquireSma(SMnode *pMnode, char *smaName) { + SSdb *pSdb = pMnode->pSdb; + SSmaObj *pSma = sdbAcquire(pSdb, SDB_SMA, smaName); + if (pSma == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { + terrno = TSDB_CODE_MND_SMA_NOT_EXIST; + } + return pSma; +} + +void mndReleaseSma(SMnode *pMnode, SSmaObj *pSma) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pSma); +} + +SDbObj *mndAcquireDbBySma(SMnode *pMnode, const char *smaName) { + SName name = {0}; + tNameFromString(&name, smaName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + + char db[TSDB_TABLE_FNAME_LEN] = {0}; + tNameGetFullDbName(&name, db); + + return mndAcquireDb(pMnode, db); +} + +static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, int32_t *pContLen) { + SName name = {0}; + tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + + SVCreateTSmaReq req = {0}; + req.tSma.version = 0; + req.tSma.intervalUnit = pSma->intervalUnit; + req.tSma.slidingUnit = pSma->slidingUnit; + req.tSma.timezoneInt = pSma->timezone; + tstrncpy(req.tSma.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN); + req.tSma.exprLen = pSma->exprLen; + req.tSma.tagsFilterLen = pSma->tagsFilterLen; + req.tSma.indexUid = pSma->uid; + req.tSma.tableUid = pSma->stbUid; + req.tSma.interval = pSma->interval; + req.tSma.offset = pSma->offset; + req.tSma.sliding = pSma->sliding; + req.tSma.expr = pSma->expr; + req.tSma.tagsFilter = pSma->tagsFilter; + + int32_t contLen = tSerializeSVCreateTSmaReq(NULL, &req) + sizeof(SMsgHead); + SMsgHead *pHead = taosMemoryMalloc(contLen); + if (pHead == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); + + void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); + tSerializeSVCreateTSmaReq(&pBuf, &req); + + *pContLen = contLen; + return pHead; +} + +static void *mndBuildVDropSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, int32_t *pContLen) { + SName name = {0}; + tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + + SVDropTSmaReq req = {0}; + req.ver = 0; + req.indexUid = pSma->uid; + tstrncpy(req.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN); + + int32_t contLen = tSerializeSVDropTSmaReq(NULL, &req) + sizeof(SMsgHead); + SMsgHead *pHead = taosMemoryMalloc(contLen); + if (pHead == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); + + void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); + tDeserializeSVDropTSmaReq(&pBuf, &req); + + *pContLen = contLen; + return pHead; +} + +static int32_t mndSetCreateSmaRedoLogs(SMnode *pMnode, STrans *pTrans, SSmaObj *pSma) { + SSdbRaw *pRedoRaw = mndSmaActionEncode(pSma); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; + + return 0; +} + +static int32_t mndSetCreateSmaCommitLogs(SMnode *pMnode, STrans *pTrans, SSmaObj *pSma) { + SSdbRaw *pCommitRaw = mndSmaActionEncode(pSma); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + + return 0; +} + +static int32_t mndSetCreateSmaRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SSmaObj *pSma) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + int32_t contLen; + + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pDb->uid) { + sdbRelease(pSdb, pVgroup); + continue; + } + + void *pReq = mndBuildVCreateSmaReq(pMnode, pVgroup, pSma, &contLen); + if (pReq == NULL) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_VND_CREATE_SMA; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} + +static int32_t mndCreateSma(SMnode *pMnode, SNodeMsg *pReq, SMCreateSmaReq *pCreate, SDbObj *pDb, SStbObj *pStb) { + SSmaObj smaObj = {0}; + memcpy(smaObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN); + memcpy(smaObj.stb, pStb->name, TSDB_TABLE_FNAME_LEN); + memcpy(smaObj.db, pDb->name, TSDB_DB_FNAME_LEN); + smaObj.createdTime = taosGetTimestampMs(); + smaObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); + smaObj.stbUid = pStb->uid; + smaObj.dbUid = pStb->dbUid; + smaObj.intervalUnit = pCreate->intervalUnit; + smaObj.slidingUnit = pCreate->slidingUnit; + smaObj.timezone = pCreate->timezone; + smaObj.dstVgId = pCreate->dstVgId; + smaObj.interval = pCreate->interval; + smaObj.offset = pCreate->offset; + smaObj.sliding = pCreate->sliding; + smaObj.exprLen = pCreate->exprLen; + smaObj.tagsFilterLen = pCreate->tagsFilterLen; + smaObj.sqlLen = pCreate->sqlLen; + smaObj.astLen = pCreate->astLen; + + if (smaObj.exprLen > 0) { + smaObj.expr = taosMemoryMalloc(smaObj.exprLen); + if (smaObj.expr == NULL) goto _OVER; + memcpy(smaObj.expr, pCreate->expr, smaObj.exprLen); + } + + if (smaObj.tagsFilterLen > 0) { + smaObj.tagsFilter = taosMemoryMalloc(smaObj.tagsFilterLen); + if (smaObj.tagsFilter == NULL) goto _OVER; + memcpy(smaObj.tagsFilter, pCreate->tagsFilter, smaObj.tagsFilterLen); + } + + if (smaObj.sqlLen > 0) { + smaObj.sql = taosMemoryMalloc(smaObj.sqlLen); + if (smaObj.sql == NULL) goto _OVER; + memcpy(smaObj.sql, pCreate->sql, smaObj.sqlLen); + } + + if (smaObj.astLen > 0) { + smaObj.ast = taosMemoryMalloc(smaObj.astLen); + if (smaObj.ast == NULL) goto _OVER; + memcpy(smaObj.ast, pCreate->ast, smaObj.astLen); + } + + SStreamObj streamObj = {0}; + tstrncpy(streamObj.name, pCreate->name, TSDB_STREAM_FNAME_LEN); + tstrncpy(streamObj.db, pDb->name, TSDB_DB_FNAME_LEN); + streamObj.createTime = taosGetTimestampMs(); + streamObj.updateTime = streamObj.createTime; + streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name)); + streamObj.dbUid = pDb->uid; + streamObj.version = 1; + streamObj.sql = pCreate->sql; + /*streamObj.physicalPlan = "";*/ + streamObj.logicalPlan = "not implemented"; + + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_SMA, &pReq->rpcMsg); + if (pTrans == NULL) goto _OVER; + + mDebug("trans:%d, used to create sma:%s", pTrans->id, pCreate->name); + mndTransSetDbInfo(pTrans, pDb); + + if (mndSetCreateSmaRedoLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER; + if (mndSetCreateSmaCommitLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER; + if (mndSetCreateSmaRedoActions(pMnode, pTrans, pDb, &smaObj) != 0) goto _OVER; + if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + + code = 0; + +_OVER: + mndTransDrop(pTrans); + return code; +} + +static int32_t mndCheckCreateSmaReq(SMCreateSmaReq *pCreate) { + terrno = TSDB_CODE_MND_INVALID_SMA_OPTION; + if (pCreate->name[0] == 0) return -1; + if (pCreate->stb[0] == 0) return -1; + if (pCreate->igExists < 0 || pCreate->igExists > 1) return -1; + if (pCreate->intervalUnit < 0) return -1; + if (pCreate->slidingUnit < 0) return -1; + if (pCreate->timezone < 0) return -1; + if (pCreate->dstVgId < 0) return -1; + if (pCreate->interval < 0) return -1; + if (pCreate->offset < 0) return -1; + if (pCreate->sliding < 0) return -1; + if (pCreate->exprLen < 0) return -1; + if (pCreate->tagsFilterLen < 0) return -1; + if (pCreate->sqlLen < 0) return -1; + if (pCreate->astLen < 0) return -1; + if (pCreate->exprLen != 0 && strlen(pCreate->expr) + 1 != pCreate->exprLen) return -1; + if (pCreate->tagsFilterLen != 0 && strlen(pCreate->tagsFilter) + 1 != pCreate->tagsFilterLen) return -1; + if (pCreate->sqlLen != 0 && strlen(pCreate->sql) + 1 != pCreate->sqlLen) return -1; + if (pCreate->astLen != 0 && strlen(pCreate->ast) + 1 != pCreate->astLen) return -1; + + SName smaName = {0}; + if (tNameFromString(&smaName, pCreate->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE) < 0) return -1; + if (*(char *)tNameGetTableName(&smaName) == 0) return -1; + + terrno = 0; + return 0; +} + +static int32_t mndProcessMCreateSmaReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; + int32_t code = -1; + SStbObj *pStb = NULL; + SSmaObj *pSma = NULL; + SStreamObj *pStream = NULL; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + SMCreateSmaReq createReq = {0}; + + if (tDeserializeSMCreateSmaReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + mDebug("sma:%s, start to create", createReq.name); + if (mndCheckCreateSmaReq(&createReq) != 0) { + goto _OVER; + } + + pStb = mndAcquireStb(pMnode, createReq.stb); + if (pStb == NULL) { + mError("sma:%s, failed to create since stb:%s not exist", createReq.name, createReq.stb); + goto _OVER; + } + + pStream = mndAcquireStream(pMnode, createReq.name); + if (pStream != NULL) { + mError("sma:%s, failed to create since stream:%s already exist", createReq.name, createReq.name); + goto _OVER; + } + + pSma = mndAcquireSma(pMnode, createReq.name); + if (pSma != NULL) { + if (createReq.igExists) { + mDebug("sma:%s, already exist in sma:%s, ignore exist is set", createReq.name, pSma->name); + code = 0; + goto _OVER; + } else { + terrno = TSDB_CODE_MND_SMA_ALREADY_EXIST; + goto _OVER; + } + } + + pDb = mndAcquireDbBySma(pMnode, createReq.name); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + goto _OVER; + } + + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto _OVER; + } + + if (mndCheckWriteAuth(pUser, pDb) != 0) { + goto _OVER; + } + + code = mndCreateSma(pMnode, pReq, &createReq, pDb, pStb); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("sma:%s, failed to create since %s", createReq.name, terrstr()); + } + + mndReleaseStb(pMnode, pStb); + mndReleaseSma(pMnode, pSma); + mndReleaseStream(pMnode, pStream); + mndReleaseDb(pMnode, pDb); + mndReleaseUser(pMnode, pUser); + tFreeSMCreateSmaReq(&createReq); + + return code; +} + +static int32_t mndProcessVCreateSmaRsp(SNodeMsg *pRsp) { + mndTransProcessRsp(pRsp); + return 0; +} + +static int32_t mndSetDropSmaRedoLogs(SMnode *pMnode, STrans *pTrans, SSmaObj *pSma) { + SSdbRaw *pRedoRaw = mndSmaActionEncode(pSma); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; + + return 0; +} + +static int32_t mndSetDropSmaCommitLogs(SMnode *pMnode, STrans *pTrans, SSmaObj *pSma) { + SSdbRaw *pCommitRaw = mndSmaActionEncode(pSma); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + + return 0; +} + +static int32_t mndSetDropSmaRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SSmaObj *pSma) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + int32_t contLen; + + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pDb->uid) { + sdbRelease(pSdb, pVgroup); + continue; + } + + int32_t contLen = 0; + void *pReq = mndBuildVDropSmaReq(pMnode, pVgroup, pSma, &contLen); + if (pReq == NULL) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_VND_DROP_SMA; + action.acceptableCode = TSDB_CODE_VND_SMA_NOT_EXIST; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} + +static int32_t mndDropSma(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pDb, SSmaObj *pSma) { + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_SMA, &pReq->rpcMsg); + if (pTrans == NULL) goto _OVER; + + mDebug("trans:%d, used to drop sma:%s", pTrans->id, pSma->name); + mndTransSetDbInfo(pTrans, pDb); + + if (mndSetDropSmaRedoLogs(pMnode, pTrans, pSma) != 0) goto _OVER; + if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER; + if (mndSetDropSmaRedoActions(pMnode, pTrans, pDb, pSma) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + + code = 0; + +_OVER: + mndTransDrop(pTrans); + return code; +} + +static int32_t mndProcessMDropSmaReq(SNodeMsg *pReq) { + SMnode *pMnode = pReq->pNode; + int32_t code = -1; + SUserObj *pUser = NULL; + SDbObj *pDb = NULL; + SSmaObj *pSma = NULL; + SMDropSmaReq dropReq = {0}; + + if (tDeserializeSMDropSmaReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + mDebug("sma:%s, start to drop", dropReq.name); + + pSma = mndAcquireSma(pMnode, dropReq.name); + if (pSma == NULL) { + if (dropReq.igNotExists) { + mDebug("sma:%s, not exist, ignore not exist is set", dropReq.name); + code = 0; + goto _OVER; + } else { + terrno = TSDB_CODE_MND_SMA_NOT_EXIST; + goto _OVER; + } + } + + pDb = mndAcquireDbBySma(pMnode, dropReq.name); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + goto _OVER; + } + + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto _OVER; + } + + if (mndCheckWriteAuth(pUser, pDb) != 0) { + goto _OVER; + } + + code = mndDropSma(pMnode, pReq, pDb, pSma); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("sma:%s, failed to drop since %s", dropReq.name, terrstr()); + } + + mndReleaseDb(pMnode, pDb); + mndReleaseSma(pMnode, pSma); + mndReleaseUser(pMnode, pUser); + + return code; +} + +static int32_t mndProcessVDropSmaRsp(SNodeMsg *pRsp) { + mndTransProcessRsp(pRsp); + return 0; +} + +static int32_t mndGetSmaMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pNode; + SSdb *pSdb = pMnode->pSdb; + + int32_t cols = 0; + SSchema *pSchema = pMeta->pSchemas; + + pShow->bytes[cols] = TSDB_INDEX_NAME_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "name"); + pSchema[cols].bytes = pShow->bytes[cols]; + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = pShow->bytes[cols]; + cols++; + + pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "stb"); + pSchema[cols].bytes = pShow->bytes[cols]; + cols++; + + pMeta->numOfColumns = cols; + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = sdbGetSize(pSdb, SDB_SMA); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tbName, mndShowStr(pShow->type)); + + return 0; +} + +static int32_t mndRetrieveSma(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pNode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SSmaObj *pSma = NULL; + int32_t cols = 0; + char *pWrite; + char prefix[TSDB_DB_FNAME_LEN] = {0}; + + SDbObj *pDb = mndAcquireDb(pMnode, pShow->db); + if (pDb == NULL) return 0; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_SMA, pShow->pIter, (void **)&pSma); + if (pShow->pIter == NULL) break; + + if (pSma->dbUid != pDb->uid) { + sdbRelease(pSdb, pSma); + continue; + } + + cols = 0; + + SName smaName = {0}; + tNameFromString(&smaName, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_TO_VARSTR(pWrite, (char *)tNameGetTableName(&smaName)); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pSma->createdTime; + cols++; + + SName stbName = {0}; + tNameFromString(&stbName, pSma->stb, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_TO_VARSTR(pWrite, (char *)tNameGetTableName(&stbName)); + cols++; + + numOfRows++; + sdbRelease(pSdb, pSma); + } + + mndReleaseDb(pMnode, pDb); + pShow->numOfReads += numOfRows; + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + return numOfRows; +} + +static void mndCancelGetNextSma(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index f01b143fbc607b2566a5f5fa9a600730f4f53314..5e0d9fae9a4bccbd382355ad2897b99c13ea8929 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -135,7 +135,7 @@ static SSdbRow *mndSnodeActionDecode(SSdbRaw *pRaw) { SNODE_DECODE_OVER: if (terrno != 0) { mError("snode:%d, failed to decode from raw:%p since %s", pObj->id, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -200,7 +200,7 @@ static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S createReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -215,7 +215,7 @@ static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } @@ -227,7 +227,7 @@ static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S dropReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -242,7 +242,7 @@ static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } @@ -352,7 +352,7 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn dropReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -367,7 +367,7 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 7b0de9d8bdcf70c94321c2af7cb614a9ee0937aa..acf45a31d946cdeb4692bd6c288b9a2259fec555 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -13,20 +13,19 @@ * along with this program. If not, see . */ -#define _DEFAULT_SOURCE #include "mndStb.h" #include "mndAuth.h" #include "mndDb.h" #include "mndDnode.h" +#include "mndInfoSchema.h" #include "mndMnode.h" #include "mndShow.h" #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" -#include "mndInfoSchema.h" #include "tname.h" -#define TSDB_STB_VER_NUMBER 1 +#define TSDB_STB_VER_NUMBER 1 #define TSDB_STB_RESERVE_SIZE 64 static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw); @@ -73,7 +72,8 @@ void mndCleanupStb(SMnode *pMnode) {} SSdbRaw *mndStbActionEncode(SStbObj *pStb) { terrno = TSDB_CODE_OUT_OF_MEMORY; - int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + TSDB_STB_RESERVE_SIZE; + int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags + pStb->numOfSmas) * sizeof(SSchema) + + TSDB_STB_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUMBER, size); if (pRaw == NULL) goto STB_ENCODE_OVER; @@ -86,8 +86,14 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { SDB_SET_INT64(pRaw, dataPos, pStb->dbUid, STB_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pStb->version, STB_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pStb->nextColId, STB_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, (int32_t)(pStb->xFilesFactor * 10000), STB_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->aggregationMethod, STB_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->delay, STB_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->ttl, STB_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns, STB_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags, STB_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->numOfSmas, STB_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->commentLen, STB_ENCODE_OVER) for (int32_t i = 0; i < pStb->numOfColumns; ++i) { SSchema *pSchema = &pStb->pColumns[i]; @@ -105,7 +111,17 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_ENCODE_OVER) } - SDB_SET_BINARY(pRaw, dataPos, pStb->comment, TSDB_STB_COMMENT_LEN, STB_ENCODE_OVER) + for (int32_t i = 0; i < pStb->numOfSmas; ++i) { + SSchema *pSchema = &pStb->pSmas[i]; + SDB_SET_INT8(pRaw, dataPos, pSchema->type, STB_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pSchema->colId, STB_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, STB_ENCODE_OVER) + SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_ENCODE_OVER) + } + + if (pStb->commentLen > 0) { + SDB_SET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen, STB_ENCODE_OVER) + } SDB_SET_RESERVE(pRaw, dataPos, TSDB_STB_RESERVE_SIZE, STB_ENCODE_OVER) SDB_SET_DATALEN(pRaw, dataPos, STB_ENCODE_OVER) @@ -148,12 +164,21 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, dataPos, &pStb->dbUid, STB_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->version, STB_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->nextColId, STB_DECODE_OVER) + int32_t xFilesFactor = 0; + SDB_GET_INT32(pRaw, dataPos, &xFilesFactor, STB_DECODE_OVER) + pStb->xFilesFactor = xFilesFactor / 10000.0f; + SDB_GET_INT32(pRaw, dataPos, &pStb->aggregationMethod, STB_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pStb->delay, STB_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pStb->ttl, STB_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->numOfColumns, STB_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->numOfTags, STB_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pStb->numOfSmas, STB_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pStb->commentLen, STB_DECODE_OVER) - pStb->pColumns = calloc(pStb->numOfColumns, sizeof(SSchema)); - pStb->pTags = calloc(pStb->numOfTags, sizeof(SSchema)); - if (pStb->pColumns == NULL || pStb->pTags == NULL) { + pStb->pColumns = taosMemoryCalloc(pStb->numOfColumns, sizeof(SSchema)); + pStb->pTags = taosMemoryCalloc(pStb->numOfTags, sizeof(SSchema)); + pStb->pSmas = taosMemoryCalloc(pStb->numOfSmas, sizeof(SSchema)); + if (pStb->pColumns == NULL || pStb->pTags == NULL || pStb->pSmas == NULL) { goto STB_DECODE_OVER; } @@ -173,7 +198,19 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_DECODE_OVER) } - SDB_GET_BINARY(pRaw, dataPos, pStb->comment, TSDB_STB_COMMENT_LEN, STB_DECODE_OVER) + for (int32_t i = 0; i < pStb->numOfSmas; ++i) { + SSchema *pSchema = &pStb->pSmas[i]; + SDB_GET_INT8(pRaw, dataPos, &pSchema->type, STB_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pSchema->colId, STB_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, STB_DECODE_OVER) + SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_DECODE_OVER) + } + + if (pStb->commentLen > 0) { + pStb->comment = taosMemoryCalloc(pStb->commentLen, 1); + if (pStb->comment == NULL) goto STB_DECODE_OVER; + SDB_GET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen, STB_DECODE_OVER) + } SDB_GET_RESERVE(pRaw, dataPos, TSDB_STB_RESERVE_SIZE, STB_DECODE_OVER) terrno = 0; @@ -181,9 +218,10 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { STB_DECODE_OVER: if (terrno != 0) { mError("stb:%s, failed to decode from raw:%p since %s", pStb->name, pRaw, terrstr()); - tfree(pStb->pColumns); - tfree(pStb->pTags); - tfree(pRow); + taosMemoryFreeClear(pStb->pColumns); + taosMemoryFreeClear(pStb->pTags); + taosMemoryFreeClear(pStb->comment); + taosMemoryFreeClear(pRow); return NULL; } @@ -198,8 +236,9 @@ static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb) { static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb) { mTrace("stb:%s, perform delete action, row:%p", pStb->name, pStb); - tfree(pStb->pColumns); - tfree(pStb->pTags); + taosMemoryFreeClear(pStb->pColumns); + taosMemoryFreeClear(pStb->pTags); + taosMemoryFreeClear(pStb->comment); return 0; } @@ -209,9 +248,9 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { taosWLockLatch(&pOld->lock); if (pOld->numOfColumns < pNew->numOfColumns) { - void *pColumns = malloc(pNew->numOfColumns * sizeof(SSchema)); + void *pColumns = taosMemoryMalloc(pNew->numOfColumns * sizeof(SSchema)); if (pColumns != NULL) { - free(pOld->pColumns); + taosMemoryFree(pOld->pColumns); pOld->pColumns = pColumns; } else { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -221,9 +260,9 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { } if (pOld->numOfTags < pNew->numOfTags) { - void *pTags = malloc(pNew->numOfTags * sizeof(SSchema)); + void *pTags = taosMemoryMalloc(pNew->numOfTags * sizeof(SSchema)); if (pTags != NULL) { - free(pOld->pTags); + taosMemoryFree(pOld->pTags); pOld->pTags = pTags; } else { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -232,6 +271,30 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { } } + if (pOld->numOfSmas < pNew->numOfSmas) { + void *pSmas = taosMemoryMalloc(pNew->numOfSmas * sizeof(SSchema)); + if (pSmas != NULL) { + taosMemoryFree(pOld->pSmas); + pOld->pSmas = pSmas; + } else { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr()); + taosWUnLockLatch(&pOld->lock); + } + } + + if (pOld->commentLen < pNew->commentLen) { + void *comment = taosMemoryMalloc(pNew->commentLen); + if (comment != NULL) { + taosMemoryFree(pOld->comment); + pOld->comment = comment; + } else { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr()); + taosWUnLockLatch(&pOld->lock); + } + } + pOld->updateTime = pNew->updateTime; pOld->version = pNew->version; pOld->nextColId = pNew->nextColId; @@ -239,7 +302,9 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { pOld->numOfTags = pNew->numOfTags; memcpy(pOld->pColumns, pNew->pColumns, pOld->numOfColumns * sizeof(SSchema)); memcpy(pOld->pTags, pNew->pTags, pOld->numOfTags * sizeof(SSchema)); - memcpy(pOld->comment, pNew->comment, TSDB_STB_COMMENT_LEN); + if (pNew->commentLen != 0) { + memcpy(pOld->comment, pNew->comment, TSDB_STB_COMMENT_LEN); + } taosWUnLockLatch(&pOld->lock); return 0; } @@ -271,9 +336,12 @@ static SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName) { static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen) { SName name = {0}; tNameFromString(&name, pStb->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(&name, dbFName); SVCreateTbReq req = {0}; req.ver = 0; + req.dbFName = dbFName; req.name = (char *)tNameGetTableName(&name); req.ttl = 0; req.keep = 0; @@ -285,7 +353,7 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt req.stbCfg.pTagSchema = pStb->pTags; int32_t contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead); - SMsgHead *pHead = malloc(contLen); + SMsgHead *pHead = taosMemoryMalloc(contLen); if (pHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -312,7 +380,7 @@ static void *mndBuildVDropStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, req.suid = pStb->uid; int32_t contLen = tSerializeSVDropTbReq(NULL, &req) + sizeof(SMsgHead); - SMsgHead *pHead = malloc(contLen); + SMsgHead *pHead = taosMemoryMalloc(contLen); if (pHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -440,7 +508,7 @@ static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.contLen = contLen; action.msgType = TDMT_VND_CREATE_STB; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); return -1; @@ -479,7 +547,7 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.contLen = contLen; action.msgType = TDMT_VND_DROP_STB; if (mndTransAppendUndoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); return -1; @@ -490,6 +558,16 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } +static SSchema *mndFindStbColumns(const SStbObj *pStb, const char *colName) { + for (int32_t col = 0; col < pStb->numOfColumns; col++) { + SSchema *pSchema = &pStb->pColumns[col]; + if (strcasecmp(pStb->pColumns[col].name, colName) == 0) { + return pSchema; + } + } + return NULL; +} + static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) { SStbObj stbObj = {0}; memcpy(stbObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN); @@ -500,12 +578,24 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre stbObj.dbUid = pDb->uid; stbObj.version = 1; stbObj.nextColId = 1; + stbObj.ttl = pCreate->ttl; stbObj.numOfColumns = pCreate->numOfColumns; stbObj.numOfTags = pCreate->numOfTags; + stbObj.numOfSmas = pCreate->numOfSmas; + stbObj.commentLen = pCreate->commentLen; + if (stbObj.commentLen > 0) { + stbObj.comment = taosMemoryCalloc(stbObj.commentLen, 1); + if (stbObj.comment == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + memcpy(stbObj.comment, pCreate->comment, stbObj.commentLen); + } - stbObj.pColumns = malloc(stbObj.numOfColumns * sizeof(SSchema)); - stbObj.pTags = malloc(stbObj.numOfTags * sizeof(SSchema)); - if (stbObj.pColumns == NULL || stbObj.pTags == NULL) { + stbObj.pColumns = taosMemoryMalloc(stbObj.numOfColumns * sizeof(SSchema)); + stbObj.pTags = taosMemoryMalloc(stbObj.numOfTags * sizeof(SSchema)); + stbObj.pSmas = taosMemoryMalloc(stbObj.numOfSmas * sizeof(SSchema)); + if (stbObj.pColumns == NULL || stbObj.pTags == NULL || stbObj.pSmas == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -530,6 +620,18 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre stbObj.nextColId++; } + for (int32_t i = 0; i < stbObj.numOfSmas; ++i) { + SField *pField = taosArrayGet(pCreate->pSmas, i); + SSchema *pSchema = &stbObj.pSmas[i]; + SSchema *pColSchema = mndFindStbColumns(&stbObj, pField->name); + if (pColSchema == NULL) { + mError("stb:%s, sma:%s not found in columns", stbObj.name, pSchema->name); + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; + return -1; + } + memcpy(pSchema, pColSchema, sizeof(SSchema)); + } + int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STB, &pReq->rpcMsg); if (pTrans == NULL) goto CREATE_STB_OVER; @@ -675,8 +777,8 @@ static int32_t mndFindSuperTableColumnIndex(const SStbObj *pStb, const char *col } static int32_t mndAllocStbSchemas(const SStbObj *pOld, SStbObj *pNew) { - pNew->pTags = calloc(pNew->numOfTags, sizeof(SSchema)); - pNew->pColumns = calloc(pNew->numOfColumns, sizeof(SSchema)); + pNew->pTags = taosMemoryCalloc(pNew->numOfTags, sizeof(SSchema)); + pNew->pColumns = taosMemoryCalloc(pNew->numOfColumns, sizeof(SSchema)); if (pNew->pTags == NULL || pNew->pColumns == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -969,7 +1071,7 @@ static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.contLen = contLen; action.msgType = TDMT_VND_ALTER_STB; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); return -1; @@ -1038,8 +1140,8 @@ static int32_t mndAlterStb(SMnode *pMnode, SNodeMsg *pReq, const SMAltertbReq *p ALTER_STB_OVER: mndTransDrop(pTrans); - tfree(stbObj.pTags); - tfree(stbObj.pColumns); + taosMemoryFreeClear(stbObj.pTags); + taosMemoryFreeClear(stbObj.pColumns); return code; } @@ -1149,7 +1251,7 @@ static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * action.msgType = TDMT_VND_DROP_STB; action.acceptableCode = TSDB_CODE_VND_TB_NOT_EXIST; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); return -1; @@ -1162,7 +1264,7 @@ static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * static int32_t mndDropStb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pDb, SStbObj *pStb) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK,TRN_TYPE_DROP_STB, &pReq->rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_STB, &pReq->rpcMsg); if (pTrans == NULL) goto DROP_STB_OVER; mDebug("trans:%d, used to drop stb:%s", pTrans->id, pStb->name); @@ -1246,7 +1348,7 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa taosRLockLatch(&pStb->lock); int32_t totalCols = pStb->numOfColumns + pStb->numOfTags; - pRsp->pSchemas = calloc(totalCols, sizeof(SSchema)); + pRsp->pSchemas = taosMemoryCalloc(totalCols, sizeof(SSchema)); if (pRsp->pSchemas == NULL) { taosRUnLockLatch(&pStb->lock); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -1396,7 +1498,7 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbVersions, int return -1; } - void *pRsp = malloc(rspLen); + void *pRsp = taosMemoryMalloc(rspLen); if (pRsp == NULL) { tFreeSTableMetaBatchRsp(&batchMetaRsp); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -1514,19 +1616,11 @@ static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, char *data, int32 if (pDb == NULL) return 0; } - tstrncpy(prefix, pShow->db, TSDB_DB_FNAME_LEN); - strcat(prefix, TS_PATH_DELIMITER); - int32_t prefixLen = (int32_t)strlen(prefix); - while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_STB, pShow->pIter, (void **)&pStb); if (pShow->pIter == NULL) break; if (pDb != NULL && pStb->dbUid != pDb->uid) { - if (strncmp(pStb->db, pDb->name, prefixLen) == 0) { - mError("Inconsistent table data, name:%s, db:%s, dbUid:%" PRIu64, pStb->name, pDb->name, pDb->uid); - } - sdbRelease(pSdb, pStb); continue; } @@ -1568,7 +1662,11 @@ static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, char *data, int32 cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_TO_VARSTR(pWrite, pStb->comment); + if (pStb->commentLen != 0) { + STR_TO_VARSTR(pWrite, pStb->comment); + } else { + STR_TO_VARSTR(pWrite, ""); + } cols++; numOfRows++; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 99aded0292e973b4c457eae5c65dd27bc13db626..dd62bc0364eb35b8cd4697c919afde86d9e5f1ad 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -14,6 +14,7 @@ */ #include "mndStream.h" +#include "parser.h" #include "mndAuth.h" #include "mndDb.h" #include "mndDnode.h" @@ -21,6 +22,7 @@ #include "mndScheduler.h" #include "mndShow.h" #include "mndStb.h" +#include "mndTopic.h" #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" @@ -33,6 +35,7 @@ static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream); static int32_t mndProcessCreateStreamReq(SNodeMsg *pReq); +static int32_t mndProcessTaskDeployInternalRsp(SNodeMsg *pRsp); /*static int32_t mndProcessDropStreamReq(SNodeMsg *pReq);*/ /*static int32_t mndProcessDropStreamInRsp(SNodeMsg *pRsp);*/ static int32_t mndProcessStreamMetaReq(SNodeMsg *pReq); @@ -50,6 +53,8 @@ int32_t mndInitStream(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndStreamActionDelete}; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STREAM, mndProcessCreateStreamReq); + mndSetMsgHandle(pMnode, TDMT_VND_TASK_DEPLOY_RSP, mndProcessTaskDeployInternalRsp); + mndSetMsgHandle(pMnode, TDMT_SND_TASK_DEPLOY_RSP, mndProcessTaskDeployInternalRsp); /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM, mndProcessDropStreamReq);*/ /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM_RSP, mndProcessDropStreamInRsp);*/ @@ -68,7 +73,7 @@ SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) { SCoder encoder; tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); - if (tEncodeSStreamObj(NULL, pStream) < 0) { + if (tEncodeSStreamObj(&encoder, pStream) < 0) { tCoderClear(&encoder); goto STREAM_ENCODE_OVER; } @@ -79,11 +84,11 @@ SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) { SSdbRaw *pRaw = sdbAllocRaw(SDB_STREAM, MND_STREAM_VER_NUMBER, size); if (pRaw == NULL) goto STREAM_ENCODE_OVER; - buf = malloc(tlen); + buf = taosMemoryMalloc(tlen); if (buf == NULL) goto STREAM_ENCODE_OVER; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, tlen, TD_ENCODER); - if (tEncodeSStreamObj(NULL, pStream) < 0) { + if (tEncodeSStreamObj(&encoder, pStream) < 0) { tCoderClear(&encoder); goto STREAM_ENCODE_OVER; } @@ -97,7 +102,7 @@ SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) { terrno = TSDB_CODE_SUCCESS; STREAM_ENCODE_OVER: - tfree(buf); + taosMemoryFreeClear(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("stream:%s, failed to encode to raw:%p since %s", pStream->name, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -130,12 +135,12 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) { int32_t tlen; int32_t dataPos = 0; SDB_GET_INT32(pRaw, dataPos, &tlen, STREAM_DECODE_OVER); - buf = malloc(tlen + 1); + buf = taosMemoryMalloc(tlen + 1); if (buf == NULL) goto STREAM_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, buf, tlen, STREAM_DECODE_OVER); SCoder decoder; - tCoderInit(&decoder, TD_LITTLE_ENDIAN, NULL, 0, TD_DECODER); + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, tlen + 1, TD_DECODER); if (tDecodeSStreamObj(&decoder, pStream) < 0) { goto STREAM_DECODE_OVER; } @@ -143,10 +148,10 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_SUCCESS; STREAM_DECODE_OVER: - tfree(buf); + taosMemoryFreeClear(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("stream:%s, failed to decode from raw:%p since %s", pStream->name, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -191,6 +196,11 @@ void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream) { sdbRelease(pSdb, pStream); } +static int32_t mndProcessTaskDeployInternalRsp(SNodeMsg *pRsp) { + mndTransProcessRsp(pRsp); + return 0; +} + static SDbObj *mndAcquireDbByStream(SMnode *pMnode, char *streamName) { SName name = {0}; tNameFromString(&name, streamName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); @@ -209,40 +219,103 @@ static int32_t mndCheckCreateStreamReq(SCMCreateStreamReq *pCreate) { return 0; } +static int32_t mndStreamGetPlanString(const char *ast, char **pStr) { + if (NULL == ast) { + return TSDB_CODE_SUCCESS; + } + + SNode *pAst = NULL; + int32_t code = nodesStringToNode(ast, &pAst); + + SQueryPlan *pPlan = NULL; + if (TSDB_CODE_SUCCESS == code) { + SPlanContext cxt = { + .pAstRoot = pAst, + .topicQuery = false, + .streamQuery = true, + }; + code = qCreateQueryPlan(&cxt, &pPlan, NULL); + } + + if (TSDB_CODE_SUCCESS == code) { + code = nodesNodeToString(pPlan, false, pStr, NULL); + } + nodesDestroyNode(pAst); + nodesDestroyNode(pPlan); + terrno = code; + return code; +} + +int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans) { + SNode *pAst = NULL; +#if 1 // TODO: remove debug info later + printf("ast = %s\n", ast); +#endif + if (nodesStringToNode(ast, &pAst) < 0) { + return -1; + } +#if 1 + SSchemaWrapper sw = {0}; + qExtractResultSchema(pAst, (int32_t*)&sw.nCols, &sw.pSchema); + + printf("|"); + for (int i = 0; i < sw.nCols; i++) { + printf(" %15s |", (char *)sw.pSchema[i].name); + } + printf("\n=======================================================\n"); + + pStream->ColAlias = NULL; +#endif + + if (TSDB_CODE_SUCCESS != mndStreamGetPlanString(ast, &pStream->physicalPlan)) { + mError("topic:%s, failed to get plan since %s", pStream->name, terrstr()); + return -1; + } + + if (mndScheduleStream(pMnode, pTrans, pStream) < 0) { + mError("stream:%ld, schedule stream since %s", pStream->uid, terrstr()); + return -1; + } + mDebug("trans:%d, used to create stream:%s", pTrans->id, pStream->name); + + SSdbRaw *pRedoRaw = mndStreamActionEncode(pStream); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + + return 0; +} + static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamReq *pCreate, SDbObj *pDb) { mDebug("stream:%s to create", pCreate->name); SStreamObj streamObj = {0}; tstrncpy(streamObj.name, pCreate->name, TSDB_STREAM_FNAME_LEN); tstrncpy(streamObj.db, pDb->name, TSDB_DB_FNAME_LEN); + tstrncpy(streamObj.outputSTbName, pCreate->outputSTbName, TSDB_TABLE_FNAME_LEN); streamObj.createTime = taosGetTimestampMs(); streamObj.updateTime = streamObj.createTime; streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name)); streamObj.dbUid = pDb->uid; streamObj.version = 1; streamObj.sql = pCreate->sql; - streamObj.physicalPlan = ""; - streamObj.logicalPlan = ""; + /*streamObj.physicalPlan = "";*/ + streamObj.logicalPlan = "not implemented"; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STREAM, &pReq->rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_STREAM, &pReq->rpcMsg); if (pTrans == NULL) { mError("stream:%s, failed to create since %s", pCreate->name, terrstr()); return -1; } mDebug("trans:%d, used to create stream:%s", pTrans->id, pCreate->name); - if (mndScheduleStream(pMnode, pTrans, &streamObj) < 0) { - mError("stream:%ld, schedule stream since %s", streamObj.uid, terrstr()); - mndTransDrop(pTrans); - return -1; - } - - SSdbRaw *pRedoRaw = mndStreamActionEncode(&streamObj); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans) != 0) { + mError("trans:%d, failed to add stream since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index a4909dfe3794f6c604c5d12cbb0f8104e5ae07d5..08a88a19ec2f252ee91b6a260973fb8883e7544a 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -94,7 +94,7 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) { tDeleteSMqSubscribeObj(pSub); - free(pSub); + taosMemoryFree(pSub); return NULL; } @@ -102,7 +102,7 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj if (mndInitUnassignedVg(pMnode, pTopic, pSub) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; tDeleteSMqSubscribeObj(pSub); - free(pSub); + taosMemoryFree(pSub); return NULL; } #endif @@ -118,7 +118,7 @@ static int32_t mndBuildRebalanceMsg(void **pBuf, int32_t *pLen, const SMqConsume }; int32_t tlen = tEncodeSMqMVRebReq(NULL, &req); - void *buf = malloc(sizeof(SMsgHead) + tlen); + void *buf = taosMemoryMalloc(sizeof(SMsgHead) + tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -157,7 +157,7 @@ static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqC mndReleaseVgroup(pMnode, pVgObj); if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(buf); + taosMemoryFree(buf); return -1; } @@ -169,7 +169,7 @@ static int32_t mndBuildCancelConnReq(void **pBuf, int32_t *pLen, const SMqConsum req.consumerId = pConsumerEp->consumerId; int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); - void *buf = malloc(sizeof(SMsgHead) + tlen); + void *buf = taosMemoryMalloc(sizeof(SMsgHead) + tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -203,7 +203,7 @@ static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMq mndReleaseVgroup(pMnode, pVgObj); if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(buf); + taosMemoryFree(buf); return -1; } @@ -229,7 +229,7 @@ static int32_t mndProcessResetOffsetReq(SNodeMsg *pMsg) { SMqOffset *pOffset = &req.offsets[i]; SMqVgOffsets *pVgOffset = taosHashGet(pHash, &pOffset->vgId, sizeof(int32_t)); if (pVgOffset == NULL) { - pVgOffset = malloc(sizeof(SMqVgOffsets)); + pVgOffset = taosMemoryMalloc(sizeof(SMqVgOffsets)); if (pVgOffset == NULL) { return -1; } @@ -407,7 +407,7 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { int32_t removeSz = taosArrayGetSize(pConsumer->recentRemovedTopics); for (int32_t i = 0; i < removeSz; i++) { char *topicName = taosArrayGet(pConsumer->recentRemovedTopics, i); - free(topicName); + taosMemoryFree(topicName); } taosArrayClear(pConsumer->recentRemovedTopics); } @@ -441,7 +441,7 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { if (pIter == NULL) break; SMqRebSubscribe *pRebSub = (SMqRebSubscribe *)pIter; SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pRebSub->key); - tfree(pRebSub->key); + taosMemoryFreeClear(pRebSub->key); mInfo("mq rebalance subscription: %s", pSub->key); @@ -762,8 +762,8 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { } mndReleaseTopic(pMnode, pTopic); mndTransDrop(pTrans); - tfree(topic); - tfree(cgroup); + taosMemoryFreeClear(topic); + taosMemoryFreeClear(cgroup); } // rebalance condition2 : imbalance assignment } @@ -836,7 +836,7 @@ static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqT strcpy(req.cgroup, cgroup); strcpy(req.topicName, pTopic->name); int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); - void *buf = malloc(sizeof(SMsgHead) + tlen); + void *buf = taosMemoryMalloc(sizeof(SMsgHead) + tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -860,7 +860,7 @@ static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqT mndReleaseVgroup(pMnode, pVgObj); if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(buf); + taosMemoryFree(buf); return -1; } return 0; @@ -877,7 +877,7 @@ static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { SSdbRaw *pRaw = sdbAllocRaw(SDB_SUBSCRIBE, MND_SUBSCRIBE_VER_NUMBER, size); if (pRaw == NULL) goto SUB_ENCODE_OVER; - buf = malloc(tlen); + buf = taosMemoryMalloc(tlen); if (buf == NULL) goto SUB_ENCODE_OVER; void *abuf = buf; @@ -892,7 +892,7 @@ static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { terrno = TSDB_CODE_SUCCESS; SUB_ENCODE_OVER: - tfree(buf); + taosMemoryFreeClear(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("subscribe:%s, failed to encode to raw:%p since %s", pSub->key, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -925,7 +925,7 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; int32_t tlen; SDB_GET_INT32(pRaw, dataPos, &tlen, SUB_DECODE_OVER); - buf = malloc(tlen + 1); + buf = taosMemoryMalloc(tlen + 1); if (buf == NULL) goto SUB_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, buf, tlen, SUB_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_SUBSCRIBE_RESERVE_SIZE, SUB_DECODE_OVER); @@ -937,10 +937,10 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_SUCCESS; SUB_DECODE_OVER: - tfree(buf); + taosMemoryFreeClear(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("subscribe:%s, failed to decode from raw:%p since %s", pSub->key, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -1140,7 +1140,7 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) { } } - if (oldSub) taosArrayDestroyEx(oldSub, free); + if (oldSub) taosArrayDestroyEx(oldSub, (void (*)(void*))taosMemoryFree); // persist consumerObj SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer); diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index 7993d7df9d7c4dbb5af38f97f7ded5a980acad05..d6c1b6c94fae8bb5a1f098c1eb956579b0b6e7cb 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -88,7 +88,7 @@ static int32_t mndProcessTelemTimer(SNodeMsg* pReq) { char* pCont = mndBuildTelemetryReport(pMnode); if (pCont != NULL) { taosSendHttpReport(TELEMETRY_SERVER, TELEMETRY_PORT, pCont, strlen(pCont), HTTP_FLAT); - free(pCont); + taosMemoryFree(pCont); } taosWUnLockLatch(&pMgmt->lock); return 0; diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index bd0fd0e612affb98f3620b3f86b595ea633c0863..625c9eb733a98699df41f5766be07087cc1f40c1 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -129,11 +129,11 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pTopic->version, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER); - pTopic->sql = calloc(pTopic->sqlLen + 1, sizeof(char)); + pTopic->sql = taosMemoryCalloc(pTopic->sqlLen + 1, sizeof(char)); SDB_GET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER); - pTopic->logicalPlan = calloc(len + 1, sizeof(char)); + pTopic->logicalPlan = taosMemoryCalloc(len + 1, sizeof(char)); if (pTopic->logicalPlan == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto TOPIC_DECODE_OVER; @@ -141,9 +141,9 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, pTopic->logicalPlan, len, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER); - pTopic->physicalPlan = calloc(len + 1, sizeof(char)); + pTopic->physicalPlan = taosMemoryCalloc(len + 1, sizeof(char)); if (pTopic->physicalPlan == NULL) { - free(pTopic->logicalPlan); + taosMemoryFree(pTopic->logicalPlan); terrno = TSDB_CODE_OUT_OF_MEMORY; goto TOPIC_DECODE_OVER; } @@ -156,7 +156,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { TOPIC_DECODE_OVER: if (terrno != TSDB_CODE_SUCCESS) { mError("topic:%s, failed to decode from raw:%p since %s", pTopic->name, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -214,7 +214,7 @@ static SDbObj *mndAcquireDbByTopic(SMnode *pMnode, char *topicName) { static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMqTopicObj *pTopic) { int32_t contLen = sizeof(SDDropTopicReq); - SDDropTopicReq *pDrop = calloc(1, contLen); + SDDropTopicReq *pDrop = taosMemoryCalloc(1, contLen); if (pDrop == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -236,7 +236,7 @@ static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) { return 0; } -static int32_t mndGetPlanString(SCMCreateTopicReq *pCreate, char **pStr) { +static int32_t mndGetPlanString(const SCMCreateTopicReq *pCreate, char **pStr) { if (NULL == pCreate->ast) { return TSDB_CODE_SUCCESS; } @@ -286,7 +286,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_TOPIC, &pReq->rpcMsg); if (pTrans == NULL) { mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); - tfree(pPlanStr); + taosMemoryFreeClear(pPlanStr); return -1; } mDebug("trans:%d, used to create topic:%s", pTrans->id, pCreate->name); @@ -294,7 +294,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq SSdbRaw *pRedoRaw = mndTopicActionEncode(&topicObj); if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); - tfree(pPlanStr); + taosMemoryFreeClear(pPlanStr); mndTransDrop(pTrans); return -1; } @@ -302,12 +302,12 @@ static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - tfree(pPlanStr); + taosMemoryFreeClear(pPlanStr); mndTransDrop(pTrans); return -1; } - tfree(pPlanStr); + taosMemoryFreeClear(pPlanStr); mndTransDrop(pTrans); return 0; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index e2a6e49b56288dbb533f85e6630795d3289f2674..33142114376c21b6ca4cd435c6ab3ea82281e370 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -255,7 +255,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < redoLogNum; ++i) { SDB_GET_INT32(pRaw, dataPos, &dataLen, TRANS_DECODE_OVER) - pData = malloc(dataLen); + pData = taosMemoryMalloc(dataLen); if (pData == NULL) goto TRANS_DECODE_OVER; mTrace("raw:%p, is created", pData); SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, TRANS_DECODE_OVER); @@ -265,7 +265,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < undoLogNum; ++i) { SDB_GET_INT32(pRaw, dataPos, &dataLen, TRANS_DECODE_OVER) - pData = malloc(dataLen); + pData = taosMemoryMalloc(dataLen); if (pData == NULL) goto TRANS_DECODE_OVER; mTrace("raw:%p, is created", pData); SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, TRANS_DECODE_OVER); @@ -275,7 +275,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < commitLogNum; ++i) { SDB_GET_INT32(pRaw, dataPos, &dataLen, TRANS_DECODE_OVER) - pData = malloc(dataLen); + pData = taosMemoryMalloc(dataLen); if (pData == NULL) goto TRANS_DECODE_OVER; mTrace("raw:%p, is created", pData); SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, TRANS_DECODE_OVER); @@ -288,7 +288,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT16(pRaw, dataPos, &action.msgType, TRANS_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, TRANS_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &action.contLen, TRANS_DECODE_OVER) - action.pCont = malloc(action.contLen); + action.pCont = taosMemoryMalloc(action.contLen); if (action.pCont == NULL) goto TRANS_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, TRANS_DECODE_OVER); if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto TRANS_DECODE_OVER; @@ -300,7 +300,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT16(pRaw, dataPos, &action.msgType, TRANS_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, TRANS_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &action.contLen, TRANS_DECODE_OVER) - action.pCont = malloc(action.contLen); + action.pCont = taosMemoryMalloc(action.contLen); if (action.pCont == NULL) goto TRANS_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, TRANS_DECODE_OVER); if (taosArrayPush(pTrans->undoActions, &action) == NULL) goto TRANS_DECODE_OVER; @@ -315,9 +315,9 @@ TRANS_DECODE_OVER: if (terrno != 0) { mError("trans:%d, failed to parse from raw:%p since %s", pTrans->id, pRaw, terrstr()); mndTransDropData(pTrans); - tfree(pRow); - tfree(pData); - tfree(action.pCont); + taosMemoryFreeClear(pRow); + taosMemoryFreeClear(pData); + taosMemoryFreeClear(action.pCont); return NULL; } @@ -406,6 +406,10 @@ static const char *mndTransType(ETrnType type) { return "alter-stb"; case TRN_TYPE_DROP_STB: return "drop-stb"; + case TRN_TYPE_CREATE_SMA: + return "create-sma"; + case TRN_TYPE_DROP_SMA: + return "drop-sma"; default: return "invalid"; } @@ -424,7 +428,7 @@ static void mndTransDropData(STrans *pTrans) { mndTransDropActions(pTrans->redoActions); mndTransDropActions(pTrans->undoActions); if (pTrans->rpcRsp != NULL) { - free(pTrans->rpcRsp); + taosMemoryFree(pTrans->rpcRsp); pTrans->rpcRsp = NULL; pTrans->rpcRspLen = 0; } @@ -468,7 +472,7 @@ static void mndReleaseTrans(SMnode *pMnode, STrans *pTrans) { } STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const SRpcMsg *pReq) { - STrans *pTrans = calloc(1, sizeof(STrans)); + STrans *pTrans = taosMemoryCalloc(1, sizeof(STrans)); if (pTrans == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to create transaction since %s", terrstr()); @@ -513,7 +517,7 @@ static void mndTransDropActions(SArray *pArray) { int32_t size = taosArrayGetSize(pArray); for (int32_t i = 0; i < size; ++i) { STransAction *pAction = taosArrayGet(pArray, i); - tfree(pAction->pCont); + taosMemoryFreeClear(pAction->pCont); } taosArrayDestroy(pArray); @@ -523,7 +527,7 @@ void mndTransDrop(STrans *pTrans) { if (pTrans != NULL) { mndTransDropData(pTrans); mDebug("trans:%d, is dropped, data:%p", pTrans->id, pTrans); - tfree(pTrans); + taosMemoryFreeClear(pTrans); } } @@ -758,7 +762,7 @@ static void mndTransSendRpcRsp(STrans *pTrans) { if (rpcCont != NULL) { memcpy(rpcCont, pTrans->rpcRsp, pTrans->rpcRspLen); } - free(pTrans->rpcRsp); + taosMemoryFree(pTrans->rpcRsp); mDebug("trans:%d, send rsp, code:0x%04x stage:%d app:%p", pTrans->id, pTrans->code & 0xFFFF, pTrans->stage, pTrans->rpcAHandle); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index b7d32c01d938ca5522deefc79bf11629c16d103e..ff34c26c4acfb4c0f887187968e4dc0d8cf417a2 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -201,7 +201,7 @@ USER_DECODE_OVER: mError("user:%s, failed to decode from raw:%p since %s", pUser->user, pRaw, terrstr()); taosHashCleanup(pUser->readDbs); taosHashCleanup(pUser->writeDbs); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 1629f71c0486af7fa348c8e90fa5281213bc58e2..d36617822da4fa4deeb496c0e842eace5f59d494 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -146,7 +146,7 @@ SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { VG_DECODE_OVER: if (terrno != 0) { mError("vgId:%d, failed to decode from raw:%p since %s", pVgroup->vgId, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -218,6 +218,8 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg createReq.hashBegin = pVgroup->hashBegin; createReq.hashEnd = pVgroup->hashEnd; createReq.hashMethod = pDb->hashMethod; + createReq.numOfRetensions = pDb->cfg.numOfRetensions; + createReq.pRetensions = pDb->cfg.pRetensions; for (int32_t v = 0; v < pVgroup->replica; ++v) { SReplica *pReplica = &createReq.replicas[v]; @@ -248,7 +250,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg return NULL; } - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -272,7 +274,7 @@ void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgOb return NULL; } - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -374,7 +376,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { SArray *pArray = NULL; SVgObj *pVgroups = NULL; - pVgroups = calloc(pDb->cfg.numOfVgroups, sizeof(SVgObj)); + pVgroups = taosMemoryCalloc(pDb->cfg.numOfVgroups, sizeof(SVgObj)); if (pVgroups == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto ALLOC_VGROUP_OVER; @@ -428,7 +430,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { mDebug("db:%s, %d vgroups is alloced, replica:%d", pDb->name, pDb->cfg.numOfVgroups, pDb->cfg.replications); ALLOC_VGROUP_OVER: - if (code != 0) free(pVgroups); + if (code != 0) taosMemoryFree(pVgroups); taosArrayDestroy(pArray); return code; } diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 6400bf69f1db66ba13db55fdf8cfc84303b96853..08f0b20d319f93fb550eff3d780686ee52a5bcc4 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -28,6 +28,7 @@ #include "mndProfile.h" #include "mndQnode.h" #include "mndShow.h" +#include "mndSma.h" #include "mndSnode.h" #include "mndStb.h" #include "mndStream.h" @@ -204,6 +205,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-offset", mndInitOffset, mndCleanupOffset) != 0) return -1; if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-stb", mndInitSma, mndCleanupSma) != 0) return -1; if (mndAllocStep(pMnode, "mnode-infos", mndInitInfos, mndCleanupInfos) != 0) return -1; if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1; @@ -281,7 +283,7 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { mDebug("start to open mnode in %s", path); - SMnode *pMnode = calloc(1, sizeof(SMnode)); + SMnode *pMnode = taosMemoryCalloc(1, sizeof(SMnode)); if (pMnode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to open mnode since %s", terrstr()); @@ -293,7 +295,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { pMnode->pSteps = taosArrayInit(24, sizeof(SMnodeStep)); if (pMnode->pSteps == NULL) { - free(pMnode); + taosMemoryFree(pMnode); terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to open mnode since %s", terrstr()); return NULL; @@ -344,8 +346,8 @@ void mndClose(SMnode *pMnode) { if (pMnode != NULL) { mDebug("start to close mnode"); mndCleanupSteps(pMnode, -1); - tfree(pMnode->path); - tfree(pMnode); + taosMemoryFreeClear(pMnode->path); + taosMemoryFreeClear(pMnode); mDebug("mnode is closed"); } } @@ -356,9 +358,7 @@ int32_t mndAlter(SMnode *pMnode, const SMnodeOpt *pOption) { return 0; } -int32_t mndStart(SMnode *pMnode) { - return mndInitTimer(pMnode); -} +int32_t mndStart(SMnode *pMnode) { return mndInitTimer(pMnode); } int32_t mndProcessMsg(SNodeMsg *pMsg) { SMnode *pMnode = pMsg->pNode; @@ -409,15 +409,15 @@ void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) { } // Note: uid 0 is reserved -uint64_t mndGenerateUid(char *name, int32_t len) { +int64_t mndGenerateUid(char *name, int32_t len) { int32_t hashval = MurmurHash3_32(name, len); do { - int64_t us = taosGetTimestampUs(); - uint64_t x = (us & 0x000000FFFFFFFFFF) << 24; - uint64_t uuid = x + ((hashval & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); + int64_t us = taosGetTimestampUs(); + int64_t x = (us & 0x000000FFFFFFFFFF) << 24; + int64_t uuid = x + ((hashval & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); if (uuid) { - return uuid; + return llabs(uuid); } } while (true); } diff --git a/source/dnode/mnode/impl/test/CMakeLists.txt b/source/dnode/mnode/impl/test/CMakeLists.txt index df0510f78369fde33b45cb458337d7fe8ac33a06..61201f33c36bb762260e96e2b9e726f7a3317a52 100644 --- a/source/dnode/mnode/impl/test/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/CMakeLists.txt @@ -12,5 +12,6 @@ add_subdirectory(dnode) add_subdirectory(mnode) add_subdirectory(db) add_subdirectory(stb) +add_subdirectory(sma) add_subdirectory(func) add_subdirectory(topic) diff --git a/source/dnode/mnode/impl/test/sma/CMakeLists.txt b/source/dnode/mnode/impl/test/sma/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..943695abf3a1cc353b50e621d9d7975288b2616d --- /dev/null +++ b/source/dnode/mnode/impl/test/sma/CMakeLists.txt @@ -0,0 +1,11 @@ +aux_source_directory(. SMA_SRC) +add_executable(mnode_test_sma ${SMA_SRC}) +target_link_libraries( + mnode_test_sma + PUBLIC sut +) + +add_test( + NAME mnode_test_sma + COMMAND mnode_test_sma +) diff --git a/source/dnode/mnode/impl/test/sma/sma.cpp b/source/dnode/mnode/impl/test/sma/sma.cpp new file mode 100644 index 0000000000000000000000000000000000000000..85f6a86183e21d7c041ff4e18124f1a36a9e3037 --- /dev/null +++ b/source/dnode/mnode/impl/test/sma/sma.cpp @@ -0,0 +1,294 @@ +/** + * @file sma.cpp + * @author slguan (slguan@taosdata.com) + * @brief MNODE module sma tests + * @version 1.0 + * @date 2022-03-23 + * + * @copyright Copyright (c) 2022 + * + */ + +#include "sut.h" + +class MndTestSma : public ::testing::Test { + protected: + static void SetUpTestSuite() { test.Init("/tmp/mnode_test_sma", 9035); } + static void TearDownTestSuite() { test.Cleanup(); } + + static Testbase test; + + public: + void SetUp() override {} + void TearDown() override {} + + void* BuildCreateDbReq(const char* dbname, int32_t* pContLen); + void* BuildDropDbReq(const char* dbname, int32_t* pContLen); + void* BuildCreateStbReq(const char* stbname, int32_t* pContLen); + void* BuildDropStbReq(const char* stbname, int32_t* pContLen); + void* BuildCreateBSmaStbReq(const char* stbname, int32_t* pContLen); + void* BuildCreateTSmaReq(const char* smaname, const char* stbname, int8_t igExists, const char* expr, + const char* tagsFilter, const char* sql, const char* ast, int32_t* pContLen); + void* BuildDropTSmaReq(const char* smaname, int8_t igNotExists, int32_t* pContLen); + + void PushField(SArray* pArray, int32_t bytes, int8_t type, const char* name); +}; + +Testbase MndTestSma::test; + +void* MndTestSma::BuildCreateDbReq(const char* dbname, int32_t* pContLen) { + SCreateDbReq createReq = {0}; + strcpy(createReq.db, dbname); + createReq.numOfVgroups = 2; + createReq.cacheBlockSize = 16; + createReq.totalBlocks = 10; + createReq.daysPerFile = 10; + createReq.daysToKeep0 = 3650; + createReq.daysToKeep1 = 3650; + createReq.daysToKeep2 = 3650; + createReq.minRows = 100; + createReq.maxRows = 4096; + createReq.commitTime = 3600; + createReq.fsyncPeriod = 3000; + createReq.walLevel = 1; + createReq.precision = 0; + createReq.compression = 2; + createReq.replications = 1; + createReq.quorum = 1; + createReq.update = 0; + createReq.cacheLastRow = 0; + createReq.ignoreExist = 1; + + int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDbReq(pReq, contLen, &createReq); + + *pContLen = contLen; + return pReq; +} + +void* MndTestSma::BuildDropDbReq(const char* dbname, int32_t* pContLen) { + SDropDbReq dropdbReq = {0}; + strcpy(dropdbReq.db, dbname); + + int32_t contLen = tSerializeSDropDbReq(NULL, 0, &dropdbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDropDbReq(pReq, contLen, &dropdbReq); + + *pContLen = contLen; + return pReq; +} + +void MndTestSma::PushField(SArray* pArray, int32_t bytes, int8_t type, const char* name) { + SField field = {0}; + field.bytes = bytes; + field.type = type; + strcpy(field.name, name); + taosArrayPush(pArray, &field); +} + +void* MndTestSma::BuildCreateStbReq(const char* stbname, int32_t* pContLen) { + SMCreateStbReq createReq = {0}; + createReq.numOfColumns = 3; + createReq.numOfTags = 1; + createReq.igExists = 0; + createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField)); + createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField)); + strcpy(createReq.name, stbname); + + PushField(createReq.pColumns, 8, TSDB_DATA_TYPE_TIMESTAMP, "ts"); + PushField(createReq.pColumns, 2, TSDB_DATA_TYPE_TINYINT, "col1"); + PushField(createReq.pColumns, 8, TSDB_DATA_TYPE_BIGINT, "col2"); + PushField(createReq.pTags, 2, TSDB_DATA_TYPE_TINYINT, "tag1"); + + int32_t tlen = tSerializeSMCreateStbReq(NULL, 0, &createReq); + void* pHead = rpcMallocCont(tlen); + tSerializeSMCreateStbReq(pHead, tlen, &createReq); + tFreeSMCreateStbReq(&createReq); + *pContLen = tlen; + return pHead; +} + +void* MndTestSma::BuildCreateBSmaStbReq(const char* stbname, int32_t* pContLen) { + SMCreateStbReq createReq = {0}; + createReq.numOfColumns = 3; + createReq.numOfTags = 1; + createReq.numOfSmas = 1; + createReq.igExists = 0; + createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField)); + createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField)); + createReq.pSmas = taosArrayInit(createReq.numOfSmas, sizeof(SField)); + strcpy(createReq.name, stbname); + + PushField(createReq.pColumns, 8, TSDB_DATA_TYPE_TIMESTAMP, "ts"); + PushField(createReq.pColumns, 2, TSDB_DATA_TYPE_TINYINT, "col1"); + PushField(createReq.pColumns, 8, TSDB_DATA_TYPE_BIGINT, "col2"); + PushField(createReq.pTags, 2, TSDB_DATA_TYPE_TINYINT, "tag1"); + PushField(createReq.pSmas, 2, TSDB_DATA_TYPE_TINYINT, "col1"); + + int32_t tlen = tSerializeSMCreateStbReq(NULL, 0, &createReq); + void* pHead = rpcMallocCont(tlen); + tSerializeSMCreateStbReq(pHead, tlen, &createReq); + tFreeSMCreateStbReq(&createReq); + *pContLen = tlen; + return pHead; +} + +void* MndTestSma::BuildDropStbReq(const char* stbname, int32_t* pContLen) { + SMDropStbReq dropstbReq = {0}; + strcpy(dropstbReq.name, stbname); + + int32_t contLen = tSerializeSMDropStbReq(NULL, 0, &dropstbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMDropStbReq(pReq, contLen, &dropstbReq); + + *pContLen = contLen; + return pReq; +} + +void* MndTestSma::BuildCreateTSmaReq(const char* smaname, const char* stbname, int8_t igExists, const char* expr, + const char* tagsFilter, const char* sql, const char* ast, int32_t* pContLen) { + SMCreateSmaReq createReq = {0}; + strcpy(createReq.name, smaname); + strcpy(createReq.stb, stbname); + createReq.igExists = igExists; + createReq.intervalUnit = 1; + createReq.slidingUnit = 2; + createReq.timezone = 3; + createReq.dstVgId = 4; + createReq.interval = 10; + createReq.offset = 5; + createReq.sliding = 6; + createReq.expr = (char*)expr; + createReq.exprLen = strlen(createReq.expr) + 1; + createReq.tagsFilter = (char*)tagsFilter; + createReq.tagsFilterLen = strlen(createReq.tagsFilter) + 1; + createReq.sql = (char*)sql; + createReq.sqlLen = strlen(createReq.sql) + 1; + createReq.ast = (char*)ast; + createReq.astLen = strlen(createReq.ast) + 1; + + int32_t tlen = tSerializeSMCreateSmaReq(NULL, 0, &createReq); + void* pHead = rpcMallocCont(tlen); + tSerializeSMCreateSmaReq(pHead, tlen, &createReq); + *pContLen = tlen; + return pHead; +} + +void* MndTestSma::BuildDropTSmaReq(const char* smaname, int8_t igNotExists, int32_t* pContLen) { + SMDropSmaReq dropsmaReq = {0}; + dropsmaReq.igNotExists = igNotExists; + strcpy(dropsmaReq.name, smaname); + + int32_t contLen = tSerializeSMDropSmaReq(NULL, 0, &dropsmaReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMDropSmaReq(pReq, contLen, &dropsmaReq); + + *pContLen = contLen; + return pReq; +} + +TEST_F(MndTestSma, 01_Create_Show_Meta_Drop_Restart_Stb) { + #if 0 + const char* dbname = "1.d1"; + const char* stbname = "1.d1.stb"; + const char* smaname = "1.d1.sma"; + int32_t contLen = 0; + void* pReq; + SRpcMsg* pRsp; + + { + pReq = BuildCreateDbReq(dbname, &contLen); + pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + } + + { + pReq = BuildCreateStbReq(stbname, &contLen); + pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + test.SendShowMetaReq(TSDB_MGMT_TABLE_STB, dbname); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + } + + { + pReq = BuildCreateTSmaReq(smaname, stbname, 0, "expr", "tagsFilter", "sql", "ast", &contLen); + pRsp = test.SendReq(TDMT_MND_CREATE_SMA, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + test.SendShowMetaReq(TSDB_MGMT_TABLE_INDEX, dbname); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + } + + // restart + test.Restart(); + + { + test.SendShowMetaReq(TSDB_MGMT_TABLE_INDEX, dbname); + CHECK_META("show indexes", 3); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckBinary("sma", TSDB_INDEX_NAME_LEN); + CheckTimestamp(); + CheckBinary("stb", TSDB_TABLE_NAME_LEN); + } + + { + pReq = BuildDropTSmaReq(smaname, 0, &contLen); + pRsp = test.SendReq(TDMT_MND_DROP_SMA, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + test.SendShowMetaReq(TSDB_MGMT_TABLE_INDEX, dbname); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 0); + } +#endif +} + +TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) { + const char* dbname = "1.d1"; + const char* stbname = "1.d1.bsmastb"; + int32_t contLen = 0; + void* pReq; + SRpcMsg* pRsp; + + { + pReq = BuildCreateDbReq(dbname, &contLen); + pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + } + + { + pReq = BuildCreateBSmaStbReq(stbname, &contLen); + pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + test.SendShowMetaReq(TSDB_MGMT_TABLE_STB, dbname); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); +// CheckBinary("bsmastb", TSDB_TABLE_NAME_LEN); + } + + test.Restart(); + + { + pReq = BuildCreateBSmaStbReq(stbname, &contLen); + pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_STB_ALREADY_EXIST); + } + + { + pReq = BuildDropStbReq(stbname, &contLen); + pRsp = test.SendReq(TDMT_MND_DROP_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + test.SendShowMetaReq(TSDB_MGMT_TABLE_STB, dbname); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 0); + } + + { + pReq = BuildDropStbReq(stbname, &contLen); + pRsp = test.SendReq(TDMT_MND_DROP_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_STB_NOT_EXIST); + } +} diff --git a/source/dnode/mnode/impl/test/trans/trans.cpp b/source/dnode/mnode/impl/test/trans/trans.cpp index 9ddb3594bb92634895910032a7595b0866f2d427..b37ec85387ce7eb28ac9c2aeb6918c37b9397e67 100644 --- a/source/dnode/mnode/impl/test/trans/trans.cpp +++ b/source/dnode/mnode/impl/test/trans/trans.cpp @@ -29,7 +29,7 @@ class MndTestTrans : public ::testing::Test { char file[PATH_MAX] = "/tmp/mnode_test_trans/mnode/data/sdb.data"; TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); int32_t size = 3 * 1024 * 1024; - void* buffer = malloc(size); + void* buffer = taosMemoryMalloc(size); int32_t readLen = taosReadFile(pFile, buffer, size); if (readLen < 0 || readLen == size) { ASSERT(1); @@ -43,7 +43,7 @@ class MndTestTrans : public ::testing::Test { if (writeLen < 0 || writeLen == readLen) { ASSERT(1); } - free(buffer); + taosMemoryFree(buffer); taosFsyncFile(pFile); taosCloseFile(&pFile); taosMsleep(1000); diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index 26809ea0596c1f339c4135938094606af0cfd567..c645dae5b5e68e6dd65ce450f30d3162864fb6a5 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -21,7 +21,7 @@ static int32_t sdbCreateDir(SSdb *pSdb); SSdb *sdbInit(SSdbOpt *pOption) { mDebug("start to init sdb in %s", pOption->path); - SSdb *pSdb = calloc(1, sizeof(SSdb)); + SSdb *pSdb = taosMemoryCalloc(1, sizeof(SSdb)); if (pSdb == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to init sdb since %s", terrstr()); @@ -67,15 +67,15 @@ void sdbCleanup(SSdb *pSdb) { sdbWriteFile(pSdb); if (pSdb->currDir != NULL) { - tfree(pSdb->currDir); + taosMemoryFreeClear(pSdb->currDir); } if (pSdb->syncDir != NULL) { - tfree(pSdb->syncDir); + taosMemoryFreeClear(pSdb->syncDir); } if (pSdb->tmpDir != NULL) { - tfree(pSdb->tmpDir); + taosMemoryFreeClear(pSdb->tmpDir); } for (ESdbType i = 0; i < SDB_MAX; ++i) { @@ -102,7 +102,7 @@ void sdbCleanup(SSdb *pSdb) { mDebug("sdb table:%d is cleaned up", i); } - free(pSdb); + taosMemoryFree(pSdb); mDebug("sdb is cleaned up"); } diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 2849da5c2ec1910c42fd62a95c4eb1629c56ad71..c11025beedfa8cbb5d8e032a28cef26319ae6d9a 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -137,7 +137,7 @@ int32_t sdbReadFile(SSdb *pSdb) { int32_t readLen = 0; int64_t ret = 0; - SSdbRaw *pRaw = malloc(SDB_MAX_SIZE); + SSdbRaw *pRaw = taosMemoryMalloc(SDB_MAX_SIZE); if (pRaw == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed read file since %s", terrstr()); @@ -150,7 +150,7 @@ int32_t sdbReadFile(SSdb *pSdb) { TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { - free(pRaw); + taosMemoryFree(pRaw); terrno = TAOS_SYSTEM_ERROR(errno); mError("failed to read file:%s since %s", file, terrstr()); return 0; @@ -159,7 +159,7 @@ int32_t sdbReadFile(SSdb *pSdb) { if (sdbReadFileHead(pSdb, pFile) != 0) { mError("failed to read file:%s head since %s", file, terrstr()); pSdb->curVer = -1; - free(pRaw); + taosMemoryFree(pRaw); taosCloseFile(&pFile); return -1; } diff --git a/source/dnode/mnode/sdb/src/sdbRaw.c b/source/dnode/mnode/sdb/src/sdbRaw.c index e37559808e2b5242354be6187c07fc49a732a940..c3aaf562bef5a557762651e770b049ee3b949416 100644 --- a/source/dnode/mnode/sdb/src/sdbRaw.c +++ b/source/dnode/mnode/sdb/src/sdbRaw.c @@ -17,7 +17,7 @@ #include "sdbInt.h" SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen) { - SSdbRaw *pRaw = calloc(1, dataLen + sizeof(SSdbRaw)); + SSdbRaw *pRaw = taosMemoryCalloc(1, dataLen + sizeof(SSdbRaw)); if (pRaw == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -33,7 +33,7 @@ SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen) { void sdbFreeRaw(SSdbRaw *pRaw) { mTrace("raw:%p, is freed", pRaw); - free(pRaw); + taosMemoryFree(pRaw); } int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val) { diff --git a/source/dnode/mnode/sdb/src/sdbRow.c b/source/dnode/mnode/sdb/src/sdbRow.c index e51dd48dcdd9fff0652e13d5b94cbe961e93bfad..ac86a72155f1bd178ab6a83b8a06d0198ad31cf2 100644 --- a/source/dnode/mnode/sdb/src/sdbRow.c +++ b/source/dnode/mnode/sdb/src/sdbRow.c @@ -17,7 +17,7 @@ #include "sdbInt.h" SSdbRow *sdbAllocRow(int32_t objSize) { - SSdbRow *pRow = calloc(1, objSize + sizeof(SSdbRow)); + SSdbRow *pRow = taosMemoryCalloc(1, objSize + sizeof(SSdbRow)); if (pRow == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -46,5 +46,5 @@ void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow) { sdbPrintOper(pSdb, pRow, "freeRow"); mTrace("row:%p, is freed", pRow->pObj); - tfree(pRow); + taosMemoryFreeClear(pRow); } diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index bc1a602b52ed3218d7a912e5c001e63299d08ab4..0d1e890097e93eef19a1500eba0794ca0452d57c 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -19,14 +19,14 @@ #include "qworker.h" SQnode *qndOpen(const SQnodeOpt *pOption) { - SQnode *pQnode = calloc(1, sizeof(SQnode)); + SQnode *pQnode = taosMemoryCalloc(1, sizeof(SQnode)); if (NULL == pQnode) { qError("calloc SQnode failed"); return NULL; } if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, NULL, (void **)&pQnode->pQuery, &pOption->msgCb)) { - tfree(pQnode); + taosMemoryFreeClear(pQnode); return NULL; } @@ -37,7 +37,7 @@ SQnode *qndOpen(const SQnodeOpt *pOption) { void qndClose(SQnode *pQnode) { qWorkerDestroy((void **)&pQnode->pQuery); - free(pQnode); + taosMemoryFree(pQnode); } int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) { return 0; } diff --git a/source/dnode/snode/CMakeLists.txt b/source/dnode/snode/CMakeLists.txt index 0c63e35e875ec28f61bfe9f6f9bd1f21f9300763..f177bda47aa12a6f7899a1fda8b09a56bf3046d5 100644 --- a/source/dnode/snode/CMakeLists.txt +++ b/source/dnode/snode/CMakeLists.txt @@ -13,4 +13,5 @@ target_link_libraries( PRIVATE common PRIVATE util PRIVATE qcom + PRIVATE stream ) diff --git a/source/dnode/snode/inc/sndInt.h b/source/dnode/snode/inc/sndInt.h index 519b94cf46cc57ce3441712ffed6b3f7b2a7e02e..2802537dcd31323a703cd6a4d2222bb8fd9d3d68 100644 --- a/source/dnode/snode/inc/sndInt.h +++ b/source/dnode/snode/inc/sndInt.h @@ -22,6 +22,7 @@ #include "tmsg.h" #include "tqueue.h" #include "trpc.h" +#include "tstream.h" #include "snode.h" diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index afa59308218f57ae0bd15522675fd9c51b764421..a5937b625f11e449dea7e76cf0397e96c503f439 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -18,14 +18,14 @@ #include "tuuid.h" SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { - SSnode *pSnode = calloc(1, sizeof(SSnode)); + SSnode *pSnode = taosMemoryCalloc(1, sizeof(SSnode)); if (pSnode == NULL) { return NULL; } pSnode->msgCb = pOption->msgCb; pSnode->pMeta = sndMetaNew(); if (pSnode->pMeta == NULL) { - free(pSnode); + taosMemoryFree(pSnode); return NULL; } return pSnode; @@ -33,19 +33,19 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { void sndClose(SSnode *pSnode) { sndMetaDelete(pSnode->pMeta); - free(pSnode); + taosMemoryFree(pSnode); } int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad) { return 0; } SStreamMeta *sndMetaNew() { - SStreamMeta *pMeta = calloc(1, sizeof(SStreamMeta)); + SStreamMeta *pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta)); if (pMeta == NULL) { return NULL; } pMeta->pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); if (pMeta->pHash == NULL) { - free(pMeta); + taosMemoryFree(pMeta); return NULL; } return pMeta; @@ -53,12 +53,12 @@ SStreamMeta *sndMetaNew() { void sndMetaDelete(SStreamMeta *pMeta) { taosHashCleanup(pMeta->pHash); - free(pMeta); + taosMemoryFree(pMeta); } int32_t sndMetaDeployTask(SStreamMeta *pMeta, SStreamTask *pTask) { - for (int i = 0; i < pTask->numOfRunners; i++) { - pTask->runner[i].executor = qCreateStreamExecTaskInfo(pTask->qmsg, NULL); + for (int i = 0; i < pTask->exec.numOfRunners; i++) { + pTask->exec.runners[i].executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, NULL); } return taosHashPut(pMeta->pHash, &pTask->taskId, sizeof(int32_t), pTask, sizeof(void *)); } @@ -72,19 +72,19 @@ int32_t sndMetaRemoveTask(SStreamMeta *pMeta, int32_t taskId) { if (pTask == NULL) { return -1; } - free(pTask->qmsg); + taosMemoryFree(pTask->exec.qmsg); // TODO:free executor - free(pTask); + taosMemoryFree(pTask); return taosHashRemove(pMeta->pHash, &taskId, sizeof(int32_t)); } static int32_t sndProcessTaskExecReq(SSnode *pSnode, SRpcMsg *pMsg) { - SStreamExecMsgHead *pHead = pMsg->pCont; - int32_t taskId = pHead->streamTaskId; - SStreamTask *pTask = sndMetaGetTask(pSnode->pMeta, taskId); - if (pTask == NULL) { - return -1; - } + /*SStreamExecMsgHead *pHead = pMsg->pCont;*/ + /*int32_t taskId = pHead->streamTaskId;*/ + /*SStreamTask *pTask = sndMetaGetTask(pSnode->pMeta, taskId);*/ + /*if (pTask == NULL) {*/ + /*return -1;*/ + /*}*/ return 0; } @@ -94,7 +94,7 @@ void sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg) { // operator exec if (pMsg->msgType == TDMT_SND_TASK_DEPLOY) { void *msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - SStreamTask *pTask = malloc(sizeof(SStreamTask)); + SStreamTask *pTask = taosMemoryMalloc(sizeof(SStreamTask)); if (pTask == NULL) { ASSERT(0); return; diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index f026f8331b9f8cb32d14dbb49a3f130ee6bf23cd..bdc8a71b0483b390f4284064071fe267ba066378 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -43,6 +43,7 @@ target_link_libraries( PUBLIC wal PUBLIC scheduler PUBLIC executor + PUBLIC stream PUBLIC qworker PUBLIC sync ) diff --git a/source/dnode/vnode/inc/meta.h b/source/dnode/vnode/inc/meta.h index a48f437c977b1b691bf9b3376e34c12e8b39ca76..149aac1206344f0526edcfe0f7451525ea96845e 100644 --- a/source/dnode/vnode/inc/meta.h +++ b/source/dnode/vnode/inc/meta.h @@ -51,7 +51,7 @@ int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg); int metaDropTable(SMeta *pMeta, tb_uid_t uid); int metaCommit(SMeta *pMeta); int32_t metaCreateTSma(SMeta *pMeta, SSmaCfg *pCfg); -int32_t metaDropTSma(SMeta *pMeta, char *indexName); +int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid); // For Query STbCfg * metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid); diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index 7b93f7580d02465fe16ddf362aa558d17121e6a4..735db64263f6cb8248473c8867c8d4ff2ebea4bd 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -87,6 +87,15 @@ int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp); int tsdbPrepareCommit(STsdb *pTsdb); int tsdbCommit(STsdb *pTsdb); +/** + * @brief When submit msg received, update the relative expired window synchronously. + * + * @param pTsdb + * @param msg + * @return int32_t + */ +int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, const char *msg); + /** * @brief Insert tSma(Time-range-wise SMA) data from stream computing engine * @@ -95,10 +104,18 @@ int tsdbCommit(STsdb *pTsdb); * @return int32_t */ int32_t tsdbInsertTSmaData(STsdb *pTsdb, char *msg); -int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, int8_t smaType, char *msg); /** - * @brief Insert RSma(Time-range-wise Rollup SMA) data. + * @brief Drop tSma data and local cache. + * + * @param pTsdb + * @param indexUid + * @return int32_t + */ +int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid); + +/** + * @brief Insert RSma(Rollup SMA) data. * * @param pTsdb * @param msg @@ -107,6 +124,20 @@ int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, int8_t smaType, char *msg); int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg); // TODO: This is the basic params, and should wrap the params to a queryHandle. +/** + * @brief Get tSma(Time-range-wise SMA) data. + * + * @param pTsdb + * @param pData + * @param indexUid + * @param interval + * @param intervalUnit + * @param tableUid + * @param colId + * @param querySKey + * @param nMaxResult + * @return int32_t + */ int32_t tsdbGetTSmaData(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySKey, int32_t nMaxResult); diff --git a/source/dnode/vnode/src/inc/metaDef.h b/source/dnode/vnode/src/inc/metaDef.h index 16a53baef073813a707d12071c1138518a403af6..bc1017f0c793ced80165c8adb964a153e583acea 100644 --- a/source/dnode/vnode/src/inc/metaDef.h +++ b/source/dnode/vnode/src/inc/metaDef.h @@ -34,7 +34,7 @@ void metaCloseDB(SMeta* pMeta); int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg); int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid); int metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg); -int metaRemoveSmaFromDb(SMeta* pMeta, const char* indexName); +int metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid); // SMetaCache int metaOpenCache(SMeta* pMeta); diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index 4d4bb12a21fdbed28c6dbc7380bd9ae491a77595..f0c3f6801a189acd7b06039f9474a7527c895931 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -161,14 +161,16 @@ typedef struct { struct STQ { // the collection of groups // the handle of meta kvstore + bool writeTrigger; char* path; STqCfg* tqConfig; STqMemRef tqMemRef; STqMetaStore* tqMeta; - STqPushMgr* tqPushMgr; - SHashObj* pStreamTasks; - SWal* pWal; - SMeta* pVnodeMeta; + // STqPushMgr* tqPushMgr; + SHashObj* pStreamTasks; + SVnode* pVnode; + SWal* pWal; + SMeta* pVnodeMeta; }; typedef struct { diff --git a/source/dnode/vnode/src/inc/tsdbDef.h b/source/dnode/vnode/src/inc/tsdbDef.h index f7fdc818d0c15f72cecac633e4799d07a26f78c5..02aba95517ceb6f639b91586150692b598c62b3e 100644 --- a/source/dnode/vnode/src/inc/tsdbDef.h +++ b/source/dnode/vnode/src/inc/tsdbDef.h @@ -63,6 +63,7 @@ struct STsdb { #define REPO_ID(r) ((r)->vgId) #define REPO_CFG(r) (&(r)->config) #define REPO_FS(r) (r)->fs +#define REPO_META(r) (r)->pMeta #define REPO_TFS(r) (r)->pTfs #define IS_REPO_LOCKED(r) (r)->repoLocked #define REPO_SMA_ENV(r, t) ((TSDB_SMA_TYPE_ROLLUP == (t)) ? (r)->pRSmaEnv : (r)->pTSmaEnv) diff --git a/source/dnode/vnode/src/inc/tsdbMemory.h b/source/dnode/vnode/src/inc/tsdbMemory.h index df4df0053f97ef984c209cba3bfb5175a4b5d432..69976fc0783117777b5e89a0c33405d6cfb4f0ae 100644 --- a/source/dnode/vnode/src/inc/tsdbMemory.h +++ b/source/dnode/vnode/src/inc/tsdbMemory.h @@ -30,7 +30,7 @@ static void taosTMemset(void *ptr, int c); static FORCE_INLINE void *taosTMalloc(size_t size) { if (size <= 0) return NULL; - void *ret = malloc(size + sizeof(size_t)); + void *ret = taosMemoryMalloc(size + sizeof(size_t)); if (ret == NULL) return NULL; *(size_t *)ret = size; @@ -58,7 +58,7 @@ static FORCE_INLINE void * taosTRealloc(void *ptr, size_t size) { void * tptr = (void *)((char *)ptr - sizeof(size_t)); size_t tsize = size + sizeof(size_t); - void* tptr1 = realloc(tptr, tsize); + void* tptr1 = taosMemoryRealloc(tptr, tsize); if (tptr1 == NULL) return NULL; tptr = tptr1; @@ -69,7 +69,7 @@ static FORCE_INLINE void * taosTRealloc(void *ptr, size_t size) { static FORCE_INLINE void* taosTZfree(void* ptr) { if (ptr) { - free((void*)((char*)ptr - sizeof(size_t))); + taosMemoryFree((void*)((char*)ptr - sizeof(size_t))); } return NULL; } diff --git a/source/dnode/vnode/src/inc/tsdbSma.h b/source/dnode/vnode/src/inc/tsdbSma.h index f934b0263d36fd406cc4c80cad1b41f76e05c91a..e0170c90e72441a3a634f753ff77c15ca53c8cdc 100644 --- a/source/dnode/vnode/src/inc/tsdbSma.h +++ b/source/dnode/vnode/src/inc/tsdbSma.h @@ -16,15 +16,17 @@ #ifndef _TD_TSDB_SMA_H_ #define _TD_TSDB_SMA_H_ -typedef struct SSmaStat SSmaStat; -typedef struct SSmaEnv SSmaEnv; +#define TSDB_SMA_TEST // remove after test finished + +typedef struct SSmaStat SSmaStat; +typedef struct SSmaEnv SSmaEnv; struct SSmaEnv { TdThreadRwlock lock; - SDiskID did; - TDBEnv dbEnv; // TODO: If it's better to put it in smaIndex level? - char * path; // relative path - SSmaStat * pStat; + SDiskID did; + TDBEnv dbEnv; // TODO: If it's better to put it in smaIndex level? + char *path; // relative path + SSmaStat *pStat; }; #define SMA_ENV_LOCK(env) ((env)->lock) diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index ed9aad927771d482c519aac8cc9218f67518c607..8d256995c68fd196f8c5df96823816bd48b78cb3 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -55,6 +55,21 @@ typedef struct SVnodeMgr { TD_DLIST(SVnodeTask) queue; } SVnodeMgr; +typedef struct { + int8_t streamType; // sma or other + int8_t dstType; + int16_t padding; + int32_t smaId; + int64_t tbUid; + int64_t lastReceivedVer; + int64_t lastCommittedVer; +} SStreamSinkInfo; + +typedef struct { + SVnode* pVnode; + SHashObj* pHash; // streamId -> SStreamSinkInfo +} SSink; + extern SVnodeMgr vnodeMgr; // SVState @@ -72,8 +87,9 @@ struct SVnode { SVBufPool* pBufPool; SMeta* pMeta; STsdb* pTsdb; - STQ* pTq; SWal* pWal; + STQ* pTq; + SSink* pSink; tsem_t canCommit; SQHandle* pQuery; SMsgCb msgCb; @@ -82,12 +98,6 @@ struct SVnode { int vnodeScheduleTask(SVnodeTask* task); -int32_t vnodePutToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq); -int32_t vnodePutToVFetchQ(SVnode* pVnode, struct SRpcMsg* pReq); -int32_t vnodeSendReq(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq); -int32_t vnodeSendMnodeReq(SVnode* pVnode, struct SRpcMsg* pReq); -void vnodeSendRsp(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pRsp); - #define vFatal(...) \ do { \ if (vDebugFlag & DEBUG_FATAL) { \ @@ -177,19 +187,20 @@ int tqInit(); void tqCleanUp(); // open in each vnode -STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac); +STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, + SMemAllocatorFactory* allocFac); void tqClose(STQ*); // required by vnode -int tqPushMsg(STQ*, void* msg, tmsg_t msgType, int64_t version); +int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version); int tqCommit(STQ*); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessSetConnReq(STQ* pTq, char* msg); int32_t tqProcessRebReq(STQ* pTq, char* msg); int32_t tqProcessTaskExec(STQ* pTq, SRpcMsg* msg); - int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen); +int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index 2dd8386d7a647fc91e55b1bfe00f7af4d557c780..fdcc1652072f127d99863957b609944845572f31 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -233,7 +233,7 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { // save sma info int32_t len = tEncodeTSma(NULL, pSmaCfg); - pBuf = calloc(len, 1); + pBuf = taosMemoryCalloc(len, 1); if (pBuf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -254,12 +254,12 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { metaDBULock(pMeta->pDB); // release - tfree(pBuf); + taosMemoryFreeClear(pBuf); return 0; } -int metaRemoveSmaFromDb(SMeta *pMeta, const char *indexName) { +int metaRemoveSmaFromDb(SMeta *pMeta, int64_t indexUid) { // TODO #if 0 DBT key = {0}; @@ -297,7 +297,7 @@ static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW) { SSchema *pSchema; buf = taosDecodeFixedU32(buf, &pSW->nCols); - pSW->pSchema = (SSchema *)malloc(sizeof(SSchema) * pSW->nCols); + pSW->pSchema = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * pSW->nCols); for (int i = 0; i < pSW->nCols; i++) { pSchema = pSW->pSchema + i; buf = taosDecodeFixedI8(buf, &pSchema->type); @@ -311,7 +311,7 @@ static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW) { static SMetaDB *metaNewDB() { SMetaDB *pDB = NULL; - pDB = (SMetaDB *)calloc(1, sizeof(*pDB)); + pDB = (SMetaDB *)taosMemoryCalloc(1, sizeof(*pDB)); if (pDB == NULL) { return NULL; } @@ -328,7 +328,7 @@ static void metaFreeDB(SMetaDB *pDB) { #if IMPL_WITH_LOCK taosThreadRwlockDestroy(&pDB->rwlock); #endif - free(pDB); + taosMemoryFree(pDB); } } @@ -463,7 +463,7 @@ static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey DBT *pDbt; if (pTbCfg->type == META_CHILD_TABLE) { - // pDbt = calloc(2, sizeof(DBT)); + // pDbt = taosMemoryCalloc(2, sizeof(DBT)); // // First key is suid // pDbt[0].data = &(pTbCfg->ctbCfg.suid); @@ -545,11 +545,11 @@ static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg) { } static void metaClearTbCfg(STbCfg *pTbCfg) { - tfree(pTbCfg->name); + taosMemoryFreeClear(pTbCfg->name); if (pTbCfg->type == META_SUPER_TABLE) { tdFreeSchema(pTbCfg->stbCfg.pTagSchema); } else if (pTbCfg->type == META_CHILD_TABLE) { - tfree(pTbCfg->ctbCfg.pTag); + taosMemoryFreeClear(pTbCfg->ctbCfg.pTag); } } @@ -574,7 +574,7 @@ STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { } // Decode - pTbCfg = (STbCfg *)malloc(sizeof(*pTbCfg)); + pTbCfg = (STbCfg *)taosMemoryMalloc(sizeof(*pTbCfg)); if (pTbCfg == NULL) { return NULL; } @@ -606,7 +606,7 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { // Decode *uid = *(tb_uid_t *)(pkey.data); - pTbCfg = (STbCfg *)malloc(sizeof(*pTbCfg)); + pTbCfg = (STbCfg *)taosMemoryMalloc(sizeof(*pTbCfg)); if (pTbCfg == NULL) { return NULL; } @@ -636,13 +636,13 @@ STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) { } // Decode - pCfg = (STSma *)calloc(1, sizeof(STSma)); + pCfg = (STSma *)taosMemoryCalloc(1, sizeof(STSma)); if (pCfg == NULL) { return NULL; } if (tDecodeTSma(value.data, pCfg) == NULL) { - tfree(pCfg); + taosMemoryFreeClear(pCfg); return NULL; } @@ -675,7 +675,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo // Decode the schema pBuf = value.data; - pSW = malloc(sizeof(*pSW)); + pSW = taosMemoryMalloc(sizeof(*pSW)); metaDecodeSchema(pBuf, pSW); return pSW; @@ -689,7 +689,7 @@ SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { SMTbCursor *pTbCur = NULL; SMetaDB *pDB = pMeta->pDB; - pTbCur = (SMTbCursor *)calloc(1, sizeof(*pTbCur)); + pTbCur = (SMTbCursor *)taosMemoryCalloc(1, sizeof(*pTbCur)); if (pTbCur == NULL) { return NULL; } @@ -722,7 +722,7 @@ void metaCloseTbCursor(SMTbCursor *pTbCur) { if (pTbCur->pCur) { pTbCur->pCur->close(pTbCur->pCur); } - free(pTbCur); + taosMemoryFree(pTbCur); } } @@ -737,8 +737,8 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) { pBuf = value.data; metaDecodeTbInfo(pBuf, &tbCfg); if (tbCfg.type == META_SUPER_TABLE) { - free(tbCfg.name); - free(tbCfg.stbCfg.pTagSchema); + taosMemoryFree(tbCfg.name); + taosMemoryFree(tbCfg.stbCfg.pTagSchema); continue; } else if (tbCfg.type == META_CHILD_TABLE) { kvRowFree(tbCfg.ctbCfg.pTag); @@ -792,7 +792,7 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { SMetaDB *pDB = pMeta->pDB; int ret; - pCtbCur = (SMCtbCursor *)calloc(1, sizeof(*pCtbCur)); + pCtbCur = (SMCtbCursor *)taosMemoryCalloc(1, sizeof(*pCtbCur)); if (pCtbCur == NULL) { return NULL; } @@ -800,7 +800,7 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { pCtbCur->suid = uid; ret = pDB->pCtbIdx->cursor(pDB->pCtbIdx, NULL, &(pCtbCur->pCur), 0); if (ret != 0) { - free(pCtbCur); + taosMemoryFree(pCtbCur); return NULL; } @@ -813,7 +813,7 @@ void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) { pCtbCur->pCur->close(pCtbCur->pCur); } - free(pCtbCur); + taosMemoryFree(pCtbCur); } } @@ -849,7 +849,7 @@ SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) { SMetaDB *pDB = pMeta->pDB; int ret; - pCur = (SMSmaCursor *)calloc(1, sizeof(*pCur)); + pCur = (SMSmaCursor *)taosMemoryCalloc(1, sizeof(*pCur)); if (pCur == NULL) { return NULL; } @@ -858,7 +858,7 @@ SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) { // TODO: lock? ret = pDB->pCtbIdx->cursor(pDB->pSmaIdx, NULL, &(pCur->pCur), 0); if (ret != 0) { - free(pCur); + taosMemoryFree(pCur); return NULL; } @@ -871,7 +871,7 @@ void metaCloseSmaCurosr(SMSmaCursor *pCur) { pCur->pCur->close(pCur->pCur); } - free(pCur); + taosMemoryFree(pCur); } } @@ -896,14 +896,14 @@ const char *metaSmaCursorNext(SMSmaCursor *pCur) { STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { STSmaWrapper *pSW = NULL; - pSW = calloc(1, sizeof(*pSW)); + pSW = taosMemoryCalloc(1, sizeof(*pSW)); if (pSW == NULL) { return NULL; } SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid); if (pCur == NULL) { - free(pSW); + taosMemoryFree(pSW); return NULL; } @@ -915,11 +915,11 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { // TODO: lock? if (pCur->pCur->pget(pCur->pCur, &skey, NULL, &pval, DB_NEXT) == 0) { ++pSW->number; - STSma *tptr = (STSma *)realloc(pSW->tSma, pSW->number * sizeof(STSma)); + STSma *tptr = (STSma *)taosMemoryRealloc(pSW->tSma, pSW->number * sizeof(STSma)); if (tptr == NULL) { metaCloseSmaCurosr(pCur); tdDestroyTSmaWrapper(pSW); - tfree(pSW); + taosMemoryFreeClear(pSW); return NULL; } pSW->tSma = tptr; @@ -927,7 +927,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { if (tDecodeTSma(pBuf, pSW->tSma + pSW->number - 1) == NULL) { metaCloseSmaCurosr(pCur); tdDestroyTSmaWrapper(pSW); - tfree(pSW); + taosMemoryFreeClear(pSW); return NULL; } continue; diff --git a/source/dnode/vnode/src/meta/metaIdx.c b/source/dnode/vnode/src/meta/metaIdx.c index 881ea4f46dcac373a7fe3fad487ae487da2ab515..818da147381c46f1aab1816407861565f071bd81 100644 --- a/source/dnode/vnode/src/meta/metaIdx.c +++ b/source/dnode/vnode/src/meta/metaIdx.c @@ -121,11 +121,11 @@ int32_t metaCreateTSma(SMeta *pMeta, SSmaCfg *pCfg) { return TSDB_CODE_SUCCESS; } -int32_t metaDropTSma(SMeta *pMeta, char* indexName) { +int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid) { // TODO: Validate the cfg // TODO: add atomicity - if (metaRemoveSmaFromDb(pMeta, indexName) < 0) { + if (metaRemoveSmaFromDb(pMeta, indexUid) < 0) { // TODO: handle error return -1; } diff --git a/source/dnode/vnode/src/meta/metaMain.c b/source/dnode/vnode/src/meta/metaMain.c index ad87b2de9ec0861293290a410b004395a882a6e3..690b96bbb058a7a74b122a73d9b604da8c18d8bb 100644 --- a/source/dnode/vnode/src/meta/metaMain.c +++ b/source/dnode/vnode/src/meta/metaMain.c @@ -69,7 +69,7 @@ static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorF SMeta *pMeta; size_t psize = strlen(path); - pMeta = (SMeta *)calloc(1, sizeof(*pMeta)); + pMeta = (SMeta *)taosMemoryCalloc(1, sizeof(*pMeta)); if (pMeta == NULL) { return NULL; } @@ -88,8 +88,8 @@ static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorF static void metaFree(SMeta *pMeta) { if (pMeta) { - tfree(pMeta->path); - free(pMeta); + taosMemoryFreeClear(pMeta->path); + taosMemoryFree(pMeta); } } diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index 4a65cf277b9dd11aeb9a0ee47cea6e19f314d10c..f4b450b4a8ee326c965d78e52dcab41940b9bd5a 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -47,7 +47,7 @@ int metaOpenDB(SMeta *pMeta) { TDB * pCtbIdx; int ret; - pDb = (SMetaDB *)calloc(1, sizeof(*pDb)); + pDb = (SMetaDB *)taosMemoryCalloc(1, sizeof(*pDb)); if (pDb == NULL) { return -1; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 3af79ca4619b0445ffab23ee9245d88e1c16dfb7..9ce4f53d279bda0479b31e4deb44729db2dcd9e1 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -16,19 +16,24 @@ #include "tcompare.h" #include "tqInt.h" #include "tqMetaStore.h" +#include "tstream.h" + +void blockDebugShowData(SArray* dataBlocks); int32_t tqInit() { return tqPushMgrInit(); } void tqCleanUp() { tqPushMgrCleanUp(); } -STQ* tqOpen(const char* path, SWal* pWal, SMeta* pVnodeMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac) { - STQ* pTq = malloc(sizeof(STQ)); +STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pVnodeMeta, STqCfg* tqConfig, + SMemAllocatorFactory* allocFac) { + STQ* pTq = taosMemoryMalloc(sizeof(STQ)); if (pTq == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return NULL; } pTq->path = strdup(path); pTq->tqConfig = tqConfig; + pTq->pVnode = pVnode; pTq->pWal = pWal; pTq->pVnodeMeta = pVnodeMeta; #if 0 @@ -39,21 +44,23 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pVnodeMeta, STqCfg* tqConfig, S } #endif pTq->tqMeta = - tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer, free, 0); + tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer, (FTqDelete)taosMemoryFree, 0); if (pTq->tqMeta == NULL) { - free(pTq); + taosMemoryFree(pTq); #if 0 allocFac->destroy(allocFac, pTq->tqMemRef.pAllocator); #endif return NULL; } +#if 0 pTq->tqPushMgr = tqPushMgrOpen(); if (pTq->tqPushMgr == NULL) { // free store - free(pTq); + taosMemoryFree(pTq); return NULL; } +#endif pTq->pStreamTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); @@ -62,52 +69,25 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pVnodeMeta, STqCfg* tqConfig, S void tqClose(STQ* pTq) { if (pTq) { - tfree(pTq->path); - free(pTq); + taosMemoryFreeClear(pTq->path); + taosMemoryFree(pTq); } // TODO } -int tqPushMsg(STQ* pTq, void* msg, tmsg_t msgType, int64_t version) { +int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version) { if (msgType != TDMT_VND_SUBMIT) return 0; - - void* pIter = NULL; - - while (1) { - pIter = taosHashIterate(pTq->pStreamTasks, pIter); - if (pIter == NULL) break; - SStreamTask* pTask = (SStreamTask*)pIter; - if (!pTask->pipeSource) continue; - - int32_t workerId = 0; - void* exec = pTask->runner[workerId].executor; - qSetStreamInput(exec, msg, STREAM_DATA_TYPE_SUBMIT_BLOCK); - SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); - while (1) { - SSDataBlock* output; - uint64_t ts; - if (qExecTask(exec, &output, &ts) < 0) { - ASSERT(false); - } - if (output == NULL) { - break; - } - taosArrayPush(pRes, output); - } - if (pTask->pipeSink) { - // write back - } else { - int32_t tlen = sizeof(SStreamExecMsgHead) + tEncodeDataBlocks(NULL, pRes); - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { - return -1; - } - void* abuf = POINTER_SHIFT(buf, sizeof(SStreamExecMsgHead)); - tEncodeDataBlocks(abuf, pRes); - // serialize - // to next level - } + void* data = taosMemoryMalloc(msgLen); + if (data == NULL) { + return -1; } + memcpy(data, msg, msgLen); + SRpcMsg req = { + .msgType = TDMT_VND_STREAM_TRIGGER, + .pCont = data, + .contLen = msgLen, + }; + tmsgPutToQueue(&pTq->pVnode->msgCb, FETCH_QUEUE, &req); #if 0 void* pIter = taosHashIterate(pTq->tqPushMgr->pHash, NULL); @@ -116,7 +96,7 @@ int tqPushMsg(STQ* pTq, void* msg, tmsg_t msgType, int64_t version) { if (pusher->type == TQ_PUSHER_TYPE__STREAM) { STqStreamPusher* streamPusher = (STqStreamPusher*)pusher; // repack - STqStreamToken* token = malloc(sizeof(STqStreamToken)); + STqStreamToken* token = taosMemoryMalloc(sizeof(STqStreamToken)); if (token == NULL) { taosHashCancelIterate(pTq->tqPushMgr->pHash, pIter); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -220,9 +200,9 @@ int tqSerializeConsumer(const STqConsumer* pConsumer, STqSerializedHead** ppHead int32_t sz = tEncodeSTqConsumer(NULL, pConsumer); if (sz > (*ppHead)->ssize) { - void* tmpPtr = realloc(*ppHead, sizeof(STqSerializedHead) + sz); + void* tmpPtr = taosMemoryRealloc(*ppHead, sizeof(STqSerializedHead) + sz); if (tmpPtr == NULL) { - free(*ppHead); + taosMemoryFree(*ppHead); terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; } @@ -239,7 +219,7 @@ int tqSerializeConsumer(const STqConsumer* pConsumer, STqSerializedHead** ppHead int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsumer** ppConsumer) { const void* str = pHead->content; - *ppConsumer = calloc(1, sizeof(STqConsumer)); + *ppConsumer = taosMemoryCalloc(1, sizeof(STqConsumer)); if (*ppConsumer == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; @@ -413,7 +393,7 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { tDecodeSMqSetCVgReq(msg, &req); /*printf("vg %d set to consumer from %ld to %ld\n", req.vgId, req.oldConsumerId, req.newConsumerId);*/ - STqConsumer* pConsumer = calloc(1, sizeof(STqConsumer)); + STqConsumer* pConsumer = taosMemoryCalloc(1, sizeof(STqConsumer)); if (pConsumer == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; @@ -424,10 +404,10 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { pConsumer->consumerId = req.consumerId; pConsumer->epoch = 0; - STqTopic* pTopic = calloc(1, sizeof(STqTopic)); + STqTopic* pTopic = taosMemoryCalloc(1, sizeof(STqTopic)); if (pTopic == NULL) { taosArrayDestroy(pConsumer->topics); - free(pConsumer); + taosMemoryFree(pConsumer); return -1; } strcpy(pTopic->topicName, req.topicName); @@ -462,28 +442,35 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { } int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) { - ASSERT(parallel <= 8); - pTask->numOfRunners = parallel; + if (pTask->execType == TASK_EXEC__NONE) return 0; + + pTask->exec.numOfRunners = parallel; for (int32_t i = 0; i < parallel; i++) { STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta); SReadHandle handle = { .reader = pReadHandle, .meta = pTq->pVnodeMeta, }; - pTask->runner[i].inputHandle = pReadHandle; - pTask->runner[i].executor = qCreateStreamExecTaskInfo(pTask->qmsg, &handle); + pTask->exec.runners = taosMemoryCalloc(parallel, sizeof(SStreamRunner)); + if (pTask->exec.runners == NULL) { + return -1; + } + pTask->exec.runners[i].inputHandle = pReadHandle; + pTask->exec.runners[i].executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); } return 0; } int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) { - SStreamTask* pTask = malloc(sizeof(SStreamTask)); + SStreamTask* pTask = taosMemoryMalloc(sizeof(SStreamTask)); if (pTask == NULL) { return -1; } SCoder decoder; tCoderInit(&decoder, TD_LITTLE_ENDIAN, (uint8_t*)msg, msgLen, TD_DECODER); - tDecodeSStreamTask(&decoder, pTask); + if (tDecodeSStreamTask(&decoder, pTask) < 0) { + ASSERT(0); + } tCoderClear(&decoder); tqExpandTask(pTq, pTask, 8); @@ -492,112 +479,29 @@ int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) { return 0; } -static char* formatTimestamp(char* buf, int64_t val, int precision) { - time_t tt; - int32_t ms = 0; - if (precision == TSDB_TIME_PRECISION_NANO) { - tt = (time_t)(val / 1000000000); - ms = val % 1000000000; - } else if (precision == TSDB_TIME_PRECISION_MICRO) { - tt = (time_t)(val / 1000000); - ms = val % 1000000; - } else { - tt = (time_t)(val / 1000); - ms = val % 1000; - } - - /* comment out as it make testcases like select_with_tags.sim fail. - but in windows, this may cause the call to localtime crash if tt < 0, - need to find a better solution. - if (tt < 0) { - tt = 0; - } - */ - -#ifdef WINDOWS - if (tt < 0) tt = 0; -#endif - if (tt <= 0 && ms < 0) { - tt--; - if (precision == TSDB_TIME_PRECISION_NANO) { - ms += 1000000000; - } else if (precision == TSDB_TIME_PRECISION_MICRO) { - ms += 1000000; - } else { - ms += 1000; - } - } - - struct tm* ptm = localtime(&tt); - size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); +int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen) { + void* pIter = NULL; - if (precision == TSDB_TIME_PRECISION_NANO) { - sprintf(buf + pos, ".%09d", ms); - } else if (precision == TSDB_TIME_PRECISION_MICRO) { - sprintf(buf + pos, ".%06d", ms); - } else { - sprintf(buf + pos, ".%03d", ms); - } + while (1) { + pIter = taosHashIterate(pTq->pStreamTasks, pIter); + if (pIter == NULL) break; + SStreamTask* pTask = (SStreamTask*)pIter; - return buf; -} -void tqDebugShowSSData(SArray* dataBlocks) { - char pBuf[128]; - int32_t sz = taosArrayGetSize(dataBlocks); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGet(dataBlocks, i); - int32_t colNum = pDataBlock->info.numOfCols; - int32_t rows = pDataBlock->info.rows; - for (int32_t j = 0; j < rows; j++) { - printf("|"); - for (int32_t k = 0; k < colNum; k++) { - SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); - void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); - switch (pColInfoData->info.type) { - case TSDB_DATA_TYPE_TIMESTAMP: - formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI); - printf(" %25s |", pBuf); - break; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - printf(" %15u |", *(uint32_t*)var); - break; - } - } - printf("\n"); + if (streamExecTask(pTask, &pTq->pVnode->msgCb, data, STREAM_DATA_TYPE_SUBMIT_BLOCK, 0) < 0) { + // TODO } } + return 0; } int32_t tqProcessTaskExec(STQ* pTq, SRpcMsg* msg) { SStreamTaskExecReq* pReq = msg->pCont; + int32_t taskId = pReq->taskId; + SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); + ASSERT(pTask); - int32_t taskId = pReq->head.streamTaskId; - int32_t workerType = pReq->head.workerType; - - SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); - // assume worker id is 1 - int32_t workerId = 1; - void* exec = pTask->runner[workerId].executor; - int32_t sz = taosArrayGetSize(pReq->data); - printf("input data:\n"); - tqDebugShowSSData(pReq->data); - SArray* pRes = taosArrayInit(0, sizeof(void*)); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* input = taosArrayGet(pReq->data, i); - SSDataBlock* output; - uint64_t ts; - qSetStreamInput(exec, input, STREAM_DATA_TYPE_SSDATA_BLOCK); - if (qExecTask(exec, &output, &ts) < 0) { - ASSERT(0); - } - if (output == NULL) { - break; - } - taosArrayPush(pRes, &output); + if (streamExecTask(pTask, &pTq->pVnode->msgCb, pReq->data, STREAM_DATA_TYPE_SSDATA_BLOCK, 0) < 0) { + // TODO } - printf("output data:\n"); - tqDebugShowSSData(pRes); - return 0; } diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c index ce00c98ff92b8293cfdb52fd47ad6de479e539e3..505687755dd9ff4499590d8fee9388fb8dd09cb9 100644 --- a/source/dnode/vnode/src/tq/tqMetaStore.c +++ b/source/dnode/vnode/src/tq/tqMetaStore.c @@ -70,7 +70,7 @@ static inline int tqReadLastPage(TdFilePtr pFile, STqIdxPageBuf* pBuf) { STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, FTqDeserialize deserializer, FTqDelete deleter, int32_t tqConfigFlag) { - STqMetaStore* pMeta = calloc(1, sizeof(STqMetaStore)); + STqMetaStore* pMeta = taosMemoryCalloc(1, sizeof(STqMetaStore)); if (pMeta == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return NULL; @@ -79,10 +79,10 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F // concat data file name and index file name size_t pathLen = strlen(path); - pMeta->dirPath = malloc(pathLen + 1); + pMeta->dirPath = taosMemoryMalloc(pathLen + 1); if (pMeta->dirPath == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - free(pMeta); + taosMemoryFree(pMeta); return NULL; } strcpy(pMeta->dirPath, path); @@ -104,7 +104,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F } pMeta->pIdxFile = pIdxFile; - pMeta->unpersistHead = calloc(1, sizeof(STqMetaList)); + pMeta->unpersistHead = taosMemoryCalloc(1, sizeof(STqMetaList)); if (pMeta->unpersistHead == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return NULL; @@ -129,7 +129,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F // read idx file and load into memory STqIdxPageBuf idxBuf; - STqSerializedHead* serializedObj = malloc(TQ_PAGE_SIZE); + STqSerializedHead* serializedObj = taosMemoryMalloc(TQ_PAGE_SIZE); if (serializedObj == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; } @@ -145,7 +145,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F ASSERT(idxBuf.head.writeOffset == idxRead); // loop read every entry for (int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) { - STqMetaList* pNode = calloc(1, sizeof(STqMetaList)); + STqMetaList* pNode = taosMemoryCalloc(1, sizeof(STqMetaList)); if (pNode == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; // TODO: free memory @@ -154,7 +154,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F taosLSeekFile(pFile, pNode->handle.offset, SEEK_SET); if (allocated < pNode->handle.serializedSize) { - void* ptr = realloc(serializedObj, pNode->handle.serializedSize); + void* ptr = taosMemoryRealloc(serializedObj, pNode->handle.serializedSize); if (ptr == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; // TODO: free memory @@ -225,11 +225,11 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F if (pBucketNode->handle.valueInTxn && pBucketNode->handle.valueInTxn != TQ_DELETE_TOKEN) { pMeta->pDeleter(pBucketNode->handle.valueInTxn); } - free(pBucketNode); + taosMemoryFree(pBucketNode); } } } - free(serializedObj); + taosMemoryFree(serializedObj); return pMeta; } @@ -252,13 +252,13 @@ int32_t tqStoreClose(STqMetaStore* pMeta) { pMeta->pDeleter(pNode->handle.valueInUse); } STqMetaList* next = pNode->next; - free(pNode); + taosMemoryFree(pNode); pNode = next; } } - free(pMeta->dirPath); - free(pMeta->unpersistHead); - free(pMeta); + taosMemoryFree(pMeta->dirPath); + taosMemoryFree(pMeta->unpersistHead); + taosMemoryFree(pMeta); return 0; } @@ -277,14 +277,14 @@ int32_t tqStoreDelete(STqMetaStore* pMeta) { pMeta->pDeleter(pNode->handle.valueInUse); } STqMetaList* next = pNode->next; - free(pNode); + taosMemoryFree(pNode); pNode = next; } } - free(pMeta->unpersistHead); + taosMemoryFree(pMeta->unpersistHead); taosRemoveDir(pMeta->dirPath); - free(pMeta->dirPath); - free(pMeta); + taosMemoryFree(pMeta->dirPath); + taosMemoryFree(pMeta); return 0; } @@ -293,7 +293,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { int64_t* bufPtr = (int64_t*)idxBuf.buffer; STqMetaList* pHead = pMeta->unpersistHead; STqMetaList* pNode = pHead->unpersistNext; - STqSerializedHead* pSHead = malloc(sizeof(STqSerializedHead)); + STqSerializedHead* pSHead = taosMemoryMalloc(sizeof(STqSerializedHead)); if (pSHead == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; @@ -383,12 +383,12 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { ASSERT(pBucketNode->next == pNode); pBucketNode->next = pNode->next; } - free(pNode); + taosMemoryFree(pNode); } } // write left bytes - free(pSHead); + taosMemoryFree(pSHead); // TODO: write new version in tfile if ((char*)bufPtr != idxBuf.buffer) { int nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, idxBuf.head.writeOffset); @@ -416,7 +416,7 @@ static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* valu pNode = pNode->next; } } - STqMetaList* pNewNode = calloc(1, sizeof(STqMetaList)); + STqMetaList* pNewNode = taosMemoryCalloc(1, sizeof(STqMetaList)); if (pNewNode == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; @@ -488,7 +488,7 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va pNode = pNode->next; } } - STqMetaList* pNewNode = calloc(1, sizeof(STqMetaList)); + STqMetaList* pNewNode = taosMemoryCalloc(1, sizeof(STqMetaList)); if (pNewNode == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; @@ -504,7 +504,7 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va int32_t tqHandleMovePut(STqMetaStore* pMeta, int64_t key, void* value) { return tqHandlePutImpl(pMeta, key, value); } int32_t tqHandleCopyPut(STqMetaStore* pMeta, int64_t key, void* value, size_t vsize) { - void* vmem = malloc(vsize); + void* vmem = taosMemoryMalloc(vsize); if (vmem == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 4115cb73137b8c02269f372e0da5abfaf154a4b1..20270950cdc6ff13208f7d86bcfee48c28dac55e 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -31,7 +31,7 @@ struct STqOffsetStore { }; STqOffsetStore* STqOffsetOpen(STqOffsetCfg* pCfg) { - STqOffsetStore* pStore = malloc(sizeof(STqOffsetStore)); + STqOffsetStore* pStore = taosMemoryMalloc(sizeof(STqOffsetStore)); if (pStore == NULL) { return NULL; } diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 4186f29e2aa7111ce53f143247bbaf5f15018d66..7851c071c340aa2fb0bc65cf9cace22166f5bd0d 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -32,7 +32,7 @@ void tqPushMgrCleanUp() { } STqPushMgr* tqPushMgrOpen() { - STqPushMgr* mgr = malloc(sizeof(STqPushMgr)); + STqPushMgr* mgr = taosMemoryMalloc(sizeof(STqPushMgr)); if (mgr == NULL) { return NULL; } @@ -42,11 +42,11 @@ STqPushMgr* tqPushMgrOpen() { void tqPushMgrClose(STqPushMgr* pushMgr) { taosHashCleanup(pushMgr->pHash); - free(pushMgr); + taosMemoryFree(pushMgr); } STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl) { - STqClientPusher* clientPusher = malloc(sizeof(STqClientPusher)); + STqClientPusher* clientPusher = taosMemoryMalloc(sizeof(STqClientPusher)); if (clientPusher == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -57,7 +57,7 @@ STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t c clientPusher->ttl = ttl; if (taosHashPut(pushMgr->pHash, &consumerId, sizeof(int64_t), &clientPusher, sizeof(void*)) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; - free(clientPusher); + taosMemoryFree(clientPusher); // TODO send rsp back return NULL; } @@ -65,7 +65,7 @@ STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t c } STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet) { - STqStreamPusher* streamPusher = malloc(sizeof(STqStreamPusher)); + STqStreamPusher* streamPusher = taosMemoryMalloc(sizeof(STqStreamPusher)); if (streamPusher == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -77,7 +77,7 @@ STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet if (taosHashPut(pushMgr->pHash, &streamId, sizeof(int64_t), &streamPusher, sizeof(void*)) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; - free(streamPusher); + taosMemoryFree(streamPusher); return NULL; } return streamPusher; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 0c4b933c198bf47aa9e8590afb186d66b28c2fc8..37e7ed11ae6ed04c532bc8ed771ac9bded12942d 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -17,7 +17,7 @@ #include "vnode.h" STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) { - STqReadHandle* pReadHandle = malloc(sizeof(STqReadHandle)); + STqReadHandle* pReadHandle = taosMemoryMalloc(sizeof(STqReadHandle)); if (pReadHandle == NULL) { return NULL; } @@ -44,7 +44,7 @@ int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t if (pReadHandle->pBlock == NULL) break; pReadHandle->pBlock->uid = htobe64(pReadHandle->pBlock->uid); - pReadHandle->pBlock->tid = htonl(pReadHandle->pBlock->tid); + pReadHandle->pBlock->suid = htobe64(pReadHandle->pBlock->suid); pReadHandle->pBlock->sversion = htonl(pReadHandle->pBlock->sversion); pReadHandle->pBlock->dataLen = htonl(pReadHandle->pBlock->dataLen); pReadHandle->pBlock->schemaLen = htonl(pReadHandle->pBlock->schemaLen); @@ -126,6 +126,36 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { if (pArray == NULL) { return NULL; } + int32_t colMeta = 0; + int32_t colNeed = 0; + while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) { + SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; + int16_t colIdSchema = pColSchema->colId; + int16_t colIdNeed = *(int16_t*)taosArrayGet(pHandle->pColIdList, colNeed); + if (colIdSchema < colIdNeed) { + colMeta++; + } else if (colIdSchema > colIdNeed) { + colNeed++; + } else { + SColumnInfoData colInfo = {0}; + int sz = numOfRows * pColSchema->bytes; + colInfo.info.bytes = pColSchema->bytes; + colInfo.info.colId = pColSchema->colId; + colInfo.info.type = pColSchema->type; + + colInfo.pData = taosMemoryCalloc(1, sz); + if (colInfo.pData == NULL) { + // TODO free + taosArrayDestroy(pArray); + return NULL; + } + + blockDataEnsureColumnCapacity(&colInfo, numOfRows); + taosArrayPush(pArray, &colInfo); + colMeta++; + colNeed++; + } + } int j = 0; for (int32_t i = 0; i < colNumNeed; i++) { @@ -143,7 +173,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { colInfo.info.colId = colId; colInfo.info.type = pColSchema->type; - colInfo.pData = calloc(1, sz); + colInfo.pData = taosMemoryCalloc(1, sz); if (colInfo.pData == NULL) { // TODO free taosArrayDestroy(pArray); @@ -163,11 +193,23 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { tdSTSRowIterReset(&iter, row); // get all wanted col of that block + int32_t colTot = taosArrayGetSize(pArray); + for (int32_t i = 0; i < colTot; i++) { + SColumnInfoData* pColData = taosArrayGet(pArray, i); + SCellVal sVal = {0}; + if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) { + break; + } + memcpy(POINTER_SHIFT(pColData->pData, curRow * pColData->info.bytes), sVal.val, pColData->info.bytes); + } +#if 0 for (int32_t i = 0; i < colNumNeed; i++) { SColumnInfoData* pColData = taosArrayGet(pArray, i); STColumn* pCol = schemaColAt(pTschema, i); // TODO - ASSERT(pCol->colId == pColData->info.colId); + if(pCol->colId != pColData->info.colId) { + continue; + } // void* val = tdGetMemRowDataOfColEx(row, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx); SCellVal sVal = {0}; if (!tdSTSRowIterNext(&iter, pCol->colId, pCol->type, &sVal)) { @@ -176,6 +218,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { } memcpy(POINTER_SHIFT(pColData->pData, curRow * pCol->bytes), sVal.val, pCol->bytes); } +#endif curRow++; } return pArray; diff --git a/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c b/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c index ee279abf47ff6acfe406d9adff26bdf4dd9ed59b..c8f4cd642ac5bea567bf1a470df0a8df42e91095 100644 --- a/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c @@ -51,7 +51,7 @@ void tsdbCloseDBF(SDBFile *pDBF) { tsdbCloseBDBDb(pDBF->pDB); pDBF->pDB = NULL; } - tfree(pDBF->path); + taosMemoryFreeClear(pDBF->path); } int32_t tsdbOpenBDBEnv(DB_ENV **ppEnv, const char *path) { @@ -159,7 +159,7 @@ void *tsdbGetSmaDataByKey(SDBFile *pDBF, void* key, uint32_t keySize, uint32_t * return NULL; } - result = calloc(1, value1.size); + result = taosMemoryCalloc(1, value1.size); if (result == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 37403f1a11955cc418e92a371da15bdc57fbf624..eca0a9612f4398f9db57732450decea1975ba3d3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -403,7 +403,7 @@ static int tsdbCreateCommitIters(SCommitH *pCommith) { STbData *pTbData; pCommith->niters = SL_SIZE(pMem->pSlIdx); - pCommith->iters = (SCommitIter *)calloc(pCommith->niters, sizeof(SCommitIter)); + pCommith->iters = (SCommitIter *)taosMemoryCalloc(pCommith->niters, sizeof(SCommitIter)); if (pCommith->iters == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; @@ -424,7 +424,7 @@ static int tsdbCreateCommitIters(SCommitH *pCommith) { pCommitIter->pIter = tSkipListCreateIter(pTbData->pData); tSkipListIterNext(pCommitIter->pIter); - pCommitIter->pTable = (STable *)malloc(sizeof(STable)); + pCommitIter->pTable = (STable *)taosMemoryMalloc(sizeof(STable)); pCommitIter->pTable->uid = pTbData->uid; pCommitIter->pTable->tid = pTbData->uid; pCommitIter->pTable->pSchema = metaGetTbTSchema(pRepo->pMeta, pTbData->uid, 0); @@ -439,10 +439,10 @@ static void tsdbDestroyCommitIters(SCommitH *pCommith) { for (int i = 1; i < pCommith->niters; i++) { tSkipListDestroyIter(pCommith->iters[i].pIter); tdFreeSchema(pCommith->iters[i].pTable->pSchema); - free(pCommith->iters[i].pTable); + taosMemoryFree(pCommith->iters[i].pTable); } - free(pCommith->iters); + taosMemoryFree(pCommith->iters); pCommith->iters = NULL; pCommith->niters = 0; } @@ -985,7 +985,7 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { // SKVRecord *pRecord; // void *pBuf = NULL; -// pBuf = malloc((size_t)maxBufSize); +// pBuf = taosMemoryMalloc((size_t)maxBufSize); // if (pBuf == NULL) { // goto _err; // } @@ -1006,7 +1006,7 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { // } // if (pRecord->size > maxBufSize) { // maxBufSize = pRecord->size; -// void* tmp = realloc(pBuf, (size_t)maxBufSize); +// void* tmp = taosMemoryRealloc(pBuf, (size_t)maxBufSize); // if (tmp == NULL) { // goto _err; // } @@ -1059,7 +1059,7 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { // pfs->metaCacheComp = NULL; // } -// tfree(pBuf); +// taosMemoryFreeClear(pBuf); // ASSERT(mf.info.nDels == 0); // ASSERT(mf.info.tombSize == 0); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index fa867543b084f8614710d162fbc2eea2e7beca53..aa235f88de98e950b9bf8730538d2017325b2c6d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -134,7 +134,7 @@ static void *tsdbDecodeFSStatus(STsdb*pRepo, void *buf, SFSStatus *pStatus) { } static SFSStatus *tsdbNewFSStatus(int maxFSet) { - SFSStatus *pStatus = (SFSStatus *)calloc(1, sizeof(*pStatus)); + SFSStatus *pStatus = (SFSStatus *)taosMemoryCalloc(1, sizeof(*pStatus)); if (pStatus == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; @@ -145,7 +145,7 @@ static SFSStatus *tsdbNewFSStatus(int maxFSet) { pStatus->df = taosArrayInit(maxFSet, sizeof(SDFileSet)); if (pStatus->df == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - free(pStatus); + taosMemoryFree(pStatus); return NULL; } @@ -155,7 +155,7 @@ static SFSStatus *tsdbNewFSStatus(int maxFSet) { static SFSStatus *tsdbFreeFSStatus(SFSStatus *pStatus) { if (pStatus) { pStatus->df = taosArrayDestroy(pStatus->df); - free(pStatus); + taosMemoryFree(pStatus); } return NULL; @@ -197,7 +197,7 @@ STsdbFS *tsdbNewFS(const STsdbCfg *pCfg) { int maxFSet = TSDB_MAX_FSETS(keep, days); STsdbFS *pfs; - pfs = (STsdbFS *)calloc(1, sizeof(*pfs)); + pfs = (STsdbFS *)taosMemoryCalloc(1, sizeof(*pfs)); if (pfs == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; @@ -206,7 +206,7 @@ STsdbFS *tsdbNewFS(const STsdbCfg *pCfg) { int code = taosThreadRwlockInit(&(pfs->lock), NULL); if (code) { terrno = TAOS_SYSTEM_ERROR(code); - free(pfs); + taosMemoryFree(pfs); return NULL; } @@ -242,7 +242,7 @@ void *tsdbFreeFS(STsdbFS *pfs) { pfs->metaCache = NULL; pfs->cstatus = tsdbFreeFSStatus(pfs->cstatus); taosThreadRwlockDestroy(&(pfs->lock)); - free(pfs); + taosMemoryFree(pfs); } return NULL; @@ -853,7 +853,7 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) { // } // if (recoverMeta) { -// pBuf = malloc((size_t)maxBufSize); +// pBuf = taosMemoryMalloc((size_t)maxBufSize); // if (pBuf == NULL) { // terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; // tsdbCloseMFile(pMFile); @@ -865,7 +865,7 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) { // if (tsdbSeekMFile(pMFile, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) { // tsdbError("vgId:%d failed to seek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), // tstrerror(terrno)); -// tfree(pBuf); +// taosMemoryFreeClear(pBuf); // tsdbCloseMFile(pMFile); // return -1; // } @@ -874,7 +874,7 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) { // if (nread < 0) { // tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), // tstrerror(terrno)); -// tfree(pBuf); +// taosMemoryFreeClear(pBuf); // tsdbCloseMFile(pMFile); // return -1; // } @@ -883,7 +883,7 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) { // tsdbError("vgId:%d failed to read file %s since file corrupted, expected read:%" PRId64 " actual read:%d", // REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), pRecord->size, nread); // terrno = TSDB_CODE_TDB_FILE_CORRUPTED; -// tfree(pBuf); +// taosMemoryFreeClear(pBuf); // tsdbCloseMFile(pMFile); // return -1; // } @@ -891,7 +891,7 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) { // if (tsdbRestoreTable(pRepo, pBuf, (int)pRecord->size) < 0) { // tsdbError("vgId:%d failed to restore table, uid %" PRId64 ", since %s" PRIu64, REPO_ID(pRepo), pRecord->uid, // tstrerror(terrno)); -// tfree(pBuf); +// taosMemoryFreeClear(pBuf); // tsdbCloseMFile(pMFile); // return -1; // } @@ -903,7 +903,7 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) { // } // tsdbCloseMFile(pMFile); -// tfree(pBuf); +// taosMemoryFreeClear(pBuf); // return 0; // } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 00e97c7b614b9277040a38f2487ea620d1c2777f..271616e9c2a031cd46dad5e849771510b320d30c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -85,7 +85,7 @@ void *tsdbDecodeSMFileEx(void *buf, SMFile *pMFile) { strncpy(TSDB_FILE_FULL_NAME(pMFile), aname, TSDB_FILENAME_LEN); TSDB_FILE_SET_CLOSED(pMFile); - tfree(aname); + taosMemoryFreeClear(aname); return buf; } @@ -119,10 +119,10 @@ int tsdbCreateMFile(SMFile *pMFile, bool updateHeader) { // Try to create directory recursively char *s = strdup(TFILE_REL_NAME(&(pMFile->f))); if (tfsMkdirRecurAt(dirname(s), TSDB_FILE_LEVEL(pMFile), TSDB_FILE_ID(pMFile)) < 0) { - tfree(s); + taosMemoryFreeClear(s); return -1; } - tfree(s); + taosMemoryFreeClear(s); pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); if (pMFile->fd < 0) { @@ -352,7 +352,7 @@ static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) { buf = taosDecodeString(buf, &aname); strncpy(TSDB_FILE_FULL_NAME(pDFile), aname, TSDB_FILENAME_LEN); TSDB_FILE_SET_CLOSED(pDFile); - tfree(aname); + taosMemoryFreeClear(aname); return buf; } @@ -366,10 +366,10 @@ int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader, TSDB_FILE_T // Try to create directory recursively char *s = strdup(TSDB_FILE_REL_NAME(pDFile)); if (tfsMkdirRecurAt(pRepo->pTfs, taosDirName(s), TSDB_FILE_DID(pDFile)) < 0) { - tfree(s); + taosMemoryFreeClear(s); return -1; } - tfree(s); + taosMemoryFreeClear(s); pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pDFile->pFile == NULL) { @@ -692,7 +692,7 @@ int tsdbParseDFilename(const char *fname, int *vid, int *fid, TSDB_FILE_T *ftype } } - tfree(p); + taosMemoryFreeClear(p); return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMain.c b/source/dnode/vnode/src/tsdb/tsdbMain.c index 131e92cab60aa1e26f5689ef5e61520e503c2d84..526109f796021cc4e17704717a02ef413ff8806e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMain.c +++ b/source/dnode/vnode/src/tsdb/tsdbMain.c @@ -68,7 +68,7 @@ static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMeta *pMeta, STfs *pTfs) { STsdb *pTsdb = NULL; - pTsdb = (STsdb *)calloc(1, sizeof(STsdb)); + pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(STsdb)); if (pTsdb == NULL) { // TODO: handle error return NULL; @@ -93,8 +93,8 @@ static void tsdbFree(STsdb *pTsdb) { tsdbFreeSmaEnv(pTsdb->pRSmaEnv); tsdbFreeSmaEnv(pTsdb->pTSmaEnv); tsdbFreeFS(pTsdb->fs); - tfree(pTsdb->path); - free(pTsdb); + taosMemoryFreeClear(pTsdb->path); + taosMemoryFree(pTsdb); } } @@ -507,7 +507,7 @@ uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t magic = pFile->info.magic; char *tfname = strdup(fname); sprintf(name, "tsdb/%s/%s", TSDB_DATA_DIR_NAME, basename(tfname)); - tfree(tfname); + taosMemoryFreeClear(tfname); } else { if ((pFGroup->fileId + 1) * TSDB_FILE_TYPE_MAX - 1 < (int)eindex) { SFile *pFile = &pFGroup->files[0]; @@ -516,17 +516,17 @@ uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t magic = pFile->info.magic; char *tfname = strdup(fname); sprintf(name, "tsdb/%s/%s", TSDB_DATA_DIR_NAME, basename(tfname)); - tfree(tfname); + taosMemoryFreeClear(tfname); } else { return 0; } } } } else { // get the named file at the specified index. If not there, return 0 - fname = malloc(256); + fname = taosMemoryMalloc(256); sprintf(fname, "%s/vnode/vnode%d/%s", tfsGetPrimaryPath(pRepo->pTfs), REPO_ID(pRepo), name); if (access(fname, F_OK) != 0) { - tfree(fname); + taosMemoryFreeClear(fname); return 0; } if (*index == TSDB_META_FILE_INDEX) { // get meta file @@ -536,19 +536,19 @@ uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t sprintf(tfname, "vnode/vnode%d/tsdb/%s/%s", REPO_ID(pRepo), TSDB_DATA_DIR_NAME, basename(name)); tsdbGetFileInfoImpl(tfname, &magic, size); } - tfree(fname); + taosMemoryFreeClear(fname); return magic; } if (stat(fname, &fState) < 0) { - tfree(fname); + taosMemoryFreeClear(fname); return 0; } *size = fState.st_size; // magic = *size; - tfree(fname); + taosMemoryFreeClear(fname); return magic; #endif } @@ -674,7 +674,7 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { } static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { - STsdbRepo *pRepo = (STsdbRepo *)calloc(1, sizeof(*pRepo)); + STsdbRepo *pRepo = (STsdbRepo *)taosMemoryCalloc(1, sizeof(*pRepo)); if (pRepo == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; @@ -748,7 +748,7 @@ static void tsdbFreeRepo(STsdbRepo *pRepo) { // tsdbFreeMemTable(pRepo->imem); tsem_destroy(&(pRepo->readyToCommit)); taosThreadMutexDestroy(&pRepo->mutex); - free(pRepo); + taosMemoryFree(pRepo); } } @@ -820,7 +820,7 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea goto out; } - pBlockStatis = calloc(numColumns, sizeof(SDataStatis)); + pBlockStatis = taosMemoryCalloc(numColumns, sizeof(SDataStatis)); if (pBlockStatis == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; err = -1; @@ -886,7 +886,7 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea // save not-null column uint16_t bytes = IS_VAR_DATA_TYPE(pCol->type) ? varDataTLen(pColData) : pCol->bytes; SDataCol *pLastCol = &(pTable->lastCols[idx]); - pLastCol->pData = malloc(bytes); + pLastCol->pData = taosMemoryMalloc(bytes); pLastCol->bytes = bytes; pLastCol->colId = pCol->colId; memcpy(pLastCol->pData, value, bytes); @@ -907,7 +907,7 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea out: taosTZfree(row); - tfree(pBlockStatis); + taosMemoryFreeClear(pBlockStatis); if (err == 0 && numColumns <= pTable->restoreColumnNum) { pTable->hasRestoreLastColumn = true; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index e46d1a0ed47fc9843a6cce6f5c6f751762e717a5..58b4b4602e67a5cf653aaf420973b867e1cae056 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -25,7 +25,7 @@ static char * tsdbTbDataGetUid(const void *arg); static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row); STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb) { - STsdbMemTable *pMemTable = (STsdbMemTable *)calloc(1, sizeof(*pMemTable)); + STsdbMemTable *pMemTable = (STsdbMemTable *)taosMemoryCalloc(1, sizeof(*pMemTable)); if (pMemTable == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -38,7 +38,7 @@ STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb) { pMemTable->nRow = 0; pMemTable->pMA = pTsdb->pmaf->create(pTsdb->pmaf); if (pMemTable->pMA == NULL) { - free(pMemTable); + taosMemoryFree(pMemTable); return NULL; } @@ -47,7 +47,7 @@ STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb) { tSkipListCreate(5, TSDB_DATA_TYPE_BIGINT, sizeof(tb_uid_t), tsdbTbDataComp, SL_DISCARD_DUP_KEY, tsdbTbDataGetUid); if (pMemTable->pSlIdx == NULL) { pTsdb->pmaf->destroy(pTsdb->pmaf, pMemTable->pMA); - free(pMemTable); + taosMemoryFree(pMemTable); return NULL; } @@ -55,7 +55,7 @@ STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb) { if (pMemTable->pHashIdx == NULL) { pTsdb->pmaf->destroy(pTsdb->pmaf, pMemTable->pMA); tSkipListDestroy(pMemTable->pSlIdx); - free(pMemTable); + taosMemoryFree(pMemTable); return NULL; } @@ -69,7 +69,7 @@ void tsdbFreeMemTable(STsdb *pTsdb, STsdbMemTable *pMemTable) { if (pMemTable->pMA) { pTsdb->pmaf->destroy(pTsdb->pmaf, pMemTable->pMA); } - free(pMemTable); + taosMemoryFree(pMemTable); } } @@ -248,7 +248,7 @@ static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { if (pBlock == NULL) break; pBlock->uid = htobe64(pBlock->uid); - pBlock->tid = htonl(pBlock->tid); + pBlock->suid = htobe64(pBlock->suid); pBlock->sversion = htonl(pBlock->sversion); pBlock->dataLen = htonl(pBlock->dataLen); pBlock->schemaLen = htonl(pBlock->schemaLen); @@ -376,7 +376,7 @@ static int tsdbMemTableInsertTbData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *p } static STbData *tsdbNewTbData(tb_uid_t uid) { - STbData *pTbData = (STbData *)calloc(1, sizeof(*pTbData)); + STbData *pTbData = (STbData *)taosMemoryCalloc(1, sizeof(*pTbData)); if (pTbData == NULL) { return NULL; } @@ -397,14 +397,14 @@ static STbData *tsdbNewTbData(tb_uid_t uid) { // tkeyComparFn, skipListCreateFlags, tsdbGetTsTupleKey); // if (pTableData->pData == NULL) { // terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - // free(pTableData); + // taosMemoryFree(pTableData); // return NULL; // } pTbData->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), tkeyComparFn, SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey); if (pTbData->pData == NULL) { - free(pTbData); + taosMemoryFree(pTbData); return NULL; } @@ -414,7 +414,7 @@ static STbData *tsdbNewTbData(tb_uid_t uid) { static void tsdbFreeTbData(STbData *pTbData) { if (pTbData) { tSkipListDestroy(pTbData->pData); - free(pTbData); + taosMemoryFree(pTbData); } } @@ -582,7 +582,7 @@ int tsdbTakeMemSnapshot(STsdbRepo *pRepo, SMemSnapshot *pSnapshot, SArray *pATab pSnapshot->mem = &(pSnapshot->mtable); - pSnapshot->mem->tData = (STableData **)calloc(pSnapshot->omem->maxTables, sizeof(STableData *)); + pSnapshot->mem->tData = (STableData **)taosMemoryCalloc(pSnapshot->omem->maxTables, sizeof(STableData *)); if (pSnapshot->mem->tData == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; taosRUnLockLatch(&(pSnapshot->omem->latch)); @@ -629,7 +629,7 @@ void tsdbUnTakeMemSnapShot(STsdbRepo *pRepo, SMemSnapshot *pSnapshot) { tsdbFreeTableData(pTableData); } } - tfree(pSnapshot->mem->tData); + taosMemoryFreeClear(pSnapshot->mem->tData); tsdbUnRefMemTable(pRepo, pSnapshot->omem); } @@ -990,10 +990,10 @@ static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, STSRow* ro TSDB_WLOCK_TABLE(pTable); SDataCol *pDataCol = &(pLatestCols[idx]); if (pDataCol->pData == NULL) { - pDataCol->pData = malloc(pTCol->bytes); + pDataCol->pData = taosMemoryMalloc(pTCol->bytes); pDataCol->bytes = pTCol->bytes; } else if (pDataCol->bytes < pTCol->bytes) { - pDataCol->pData = realloc(pDataCol->pData, pTCol->bytes); + pDataCol->pData = taosMemoryRealloc(pDataCol->pData, pTCol->bytes); pDataCol->bytes = pTCol->bytes; } // the actual value size diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 5af9e18ee1ebf0f49b11a4c32392bc42045d7bf1..bac5255d17fd8de08dc19a0060321630b5c3caff 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -355,7 +355,7 @@ static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, STsdbQueryCond* } static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, uint64_t qId, uint64_t taskId) { - STsdbReadHandle* pReadHandle = calloc(1, sizeof(STsdbReadHandle)); + STsdbReadHandle* pReadHandle = taosMemoryCalloc(1, sizeof(STsdbReadHandle)); if (pReadHandle == NULL) { goto _end; } @@ -388,7 +388,7 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, if (pCond->numOfCols > 0) { // allocate buffer in order to load data blocks from file - pReadHandle->statis = calloc(pCond->numOfCols, sizeof(SDataStatis)); + pReadHandle->statis = taosMemoryCalloc(pCond->numOfCols, sizeof(SDataStatis)); if (pReadHandle->statis == NULL) { goto _end; } @@ -403,18 +403,12 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, SColumnInfoData colInfo = {{0}, 0}; colInfo.info = pCond->colList[i]; - colInfo.pData = calloc(1, EXTRA_BYTES + pReadHandle->outputCapacity * pCond->colList[i].bytes); - if (!IS_VAR_DATA_TYPE(colInfo.info.type)) { - colInfo.nullbitmap = calloc(1, BitmapLen(pReadHandle->outputCapacity)); - } - - if (colInfo.pData == NULL || (colInfo.nullbitmap == NULL && (!IS_VAR_DATA_TYPE(colInfo.info.type)))) { + int32_t code = blockDataEnsureColumnCapacity(&colInfo, pReadHandle->outputCapacity); + if (code != TSDB_CODE_SUCCESS) { goto _end; } taosArrayPush(pReadHandle->pColumns, &colInfo); - - pReadHandle->statis[i].colId = colInfo.info.colId; } @@ -603,7 +597,7 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr assert(pGroupList); size_t numOfGroup = taosArrayGetSize(pGroupList->pGroupList); - STableGroupInfo* pNew = calloc(1, sizeof(STableGroupInfo)); + STableGroupInfo* pNew = taosMemoryCalloc(1, sizeof(STableGroupInfo)); pNew->pGroupList = taosArrayInit(numOfGroup, POINTER_BYTES); for(int32_t i = 0; i < numOfGroup; ++i) { @@ -1015,7 +1009,7 @@ static int32_t loadBlockInfo(STsdbReadHandle * pTsdbReadHandle, int32_t index, i if (pCheckInfo->compSize < (int32_t)compIndex->len) { assert(compIndex->len > 0); - char* t = realloc(pCheckInfo->pCompInfo, compIndex->len); + char* t = taosMemoryRealloc(pCheckInfo->pCompInfo, compIndex->len); if (t == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; code = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -1383,7 +1377,6 @@ static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) { } static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end) { - char* pData = NULL; int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1 : -1; SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; @@ -1409,43 +1402,38 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t continue; } - int32_t bytes = pColInfo->info.bytes; - - if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; - } else { - pData = (char*)pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes; - } - if (!isAllRowsNull(src) && pColInfo->info.colId == src->colId) { - if (pColInfo->info.type != TSDB_DATA_TYPE_BINARY && pColInfo->info.type != TSDB_DATA_TYPE_NCHAR) { - memmove(pData, (char*)src->pData + bytes * start, bytes * num); - } else { // handle the var-string - char* dst = pData; + if (!IS_VAR_DATA_TYPE(pColInfo->info.type)) { // todo opt performance +// memmove(pData, (char*)src->pData + bytes * start, bytes * num); + for(int32_t k = start; k < num + start; ++k) { + SCellVal sVal = {0}; + if (tdGetColDataOfRow(&sVal, src, k) < 0) { + TASSERT(0); + } + if (sVal.valType == TD_VTYPE_NULL) { + colDataAppend(pColInfo, k, NULL, true); + } else { + colDataAppend(pColInfo, k, sVal.val, false); + } + } + } else { // handle the var-string // todo refactor, only copy one-by-one for (int32_t k = start; k < num + start; ++k) { - SCellVal sVal = {0}; + SCellVal sVal = {0}; if(tdGetColDataOfRow(&sVal, src, k) < 0){ TASSERT(0); } - memcpy(dst, sVal.val, varDataTLen(sVal.val)); - dst += bytes; + + colDataAppend(pColInfo, k, sVal.val, false); } } j++; i++; } else { // pColInfo->info.colId < src->colId, it is a NULL data - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - char* dst = pData; - - for(int32_t k = start; k < num + start; ++k) { - setVardataNull(dst, pColInfo->info.type); - dst += bytes; - } - } else { - setNullN(pData, pColInfo->info.type, pColInfo->info.bytes, num); + for(int32_t k = start; k < num + start; ++k) { // TODO opt performance + colDataAppend(pColInfo, k, NULL, true); } i++; } @@ -1453,23 +1441,9 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t while (i < requiredNumOfCols) { // the remain columns are all null data SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); - if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; - } else { - pData = (char*)pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes; - } - - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - char* dst = pData; - - for(int32_t k = start; k < num + start; ++k) { - setVardataNull(dst, pColInfo->info.type); - dst += pColInfo->info.bytes; - } - } else { - setNullN(pData, pColInfo->info.type, pColInfo->info.bytes, num); + for(int32_t k = start; k < num + start; ++k) { + colDataAppend(pColInfo, k, NULL, true); // TODO add a fast version to set a number of consecutive NULL value. } - i++; } @@ -1479,14 +1453,12 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t return numOfRows + num; } -// TODO fix bug for reverse copy data -// TODO handle the null data +// TODO fix bug for reverse copy data problem // Note: row1 always has high priority static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, STSRow* row1, STSRow* row2, int32_t numOfCols, uint64_t uid, STSchema* pSchema1, STSchema* pSchema2, bool forceSetNull) { #if 1 - char* pData = NULL; STSchema* pSchema; STSRow* row; int16_t colId; @@ -1528,12 +1500,6 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit while(i < numOfCols && (j < numOfColsOfRow1 || k < numOfColsOfRow2)) { SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); - if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; - } else { - pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; - } - int32_t colIdOfRow1; if(j >= numOfColsOfRow1) { colIdOfRow1 = INT32_MAX; @@ -1596,43 +1562,11 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit if (colId == pColInfo->info.colId) { if (tdValTypeIsNorm(sVal.valType)) { - switch (pColInfo->info.type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - memcpy(pData, sVal.val, varDataTLen(sVal.val)); - break; - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - *(uint8_t *)pData = *(uint8_t *)sVal.val; - break; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - *(uint16_t *)pData = *(uint16_t *)sVal.val; - break; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - *(uint32_t *)pData = *(uint32_t *)sVal.val; - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - *(uint64_t *)pData = *(uint64_t *)sVal.val; - break; - case TSDB_DATA_TYPE_FLOAT: - SET_FLOAT_PTR(pData, sVal.val); - break; - case TSDB_DATA_TYPE_DOUBLE: - SET_DOUBLE_PTR(pData, sVal.val); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - *(TSKEY*)pData = *(TSKEY*)sVal.val; - break; - default: - memcpy(pData, sVal.val, pColInfo->info.bytes); - } + colDataAppend(pColInfo, numOfRows, sVal.val, false); } else if (forceSetNull) { colDataAppend(pColInfo, numOfRows, NULL, true); } + i++; if(row == row1) { @@ -2040,15 +1974,15 @@ int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { } static void cleanBlockOrderSupporter(SBlockOrderSupporter* pSupporter, int32_t numOfTables) { - tfree(pSupporter->numOfBlocksPerTable); - tfree(pSupporter->blockIndexArray); + taosMemoryFreeClear(pSupporter->numOfBlocksPerTable); + taosMemoryFreeClear(pSupporter->blockIndexArray); for (int32_t i = 0; i < numOfTables; ++i) { STableBlockInfo* pBlockInfo = pSupporter->pDataBlockInfo[i]; - tfree(pBlockInfo); + taosMemoryFreeClear(pBlockInfo); } - tfree(pSupporter->pDataBlockInfo); + taosMemoryFreeClear(pSupporter->pDataBlockInfo); } static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void* param) { @@ -2087,7 +2021,7 @@ static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t nu if (pTsdbReadHandle->allocSize < size) { pTsdbReadHandle->allocSize = (int32_t)size; - char* tmp = realloc(pTsdbReadHandle->pDataBlockInfo, pTsdbReadHandle->allocSize); + char* tmp = taosMemoryRealloc(pTsdbReadHandle->pDataBlockInfo, pTsdbReadHandle->allocSize); if (tmp == NULL) { return TSDB_CODE_TDB_OUT_OF_MEMORY; } @@ -2103,9 +2037,9 @@ static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t nu SBlockOrderSupporter sup = {0}; sup.numOfTables = numOfTables; - sup.numOfBlocksPerTable = calloc(1, sizeof(int32_t) * numOfTables); - sup.blockIndexArray = calloc(1, sizeof(int32_t) * numOfTables); - sup.pDataBlockInfo = calloc(1, POINTER_BYTES * numOfTables); + sup.numOfBlocksPerTable = taosMemoryCalloc(1, sizeof(int32_t) * numOfTables); + sup.blockIndexArray = taosMemoryCalloc(1, sizeof(int32_t) * numOfTables); + sup.pDataBlockInfo = taosMemoryCalloc(1, POINTER_BYTES * numOfTables); if (sup.numOfBlocksPerTable == NULL || sup.blockIndexArray == NULL || sup.pDataBlockInfo == NULL) { cleanBlockOrderSupporter(&sup, 0); @@ -2124,7 +2058,7 @@ static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t nu SBlock* pBlock = pTableCheck->pCompInfo->blocks; sup.numOfBlocksPerTable[numOfQualTables] = pTableCheck->numOfBlocks; - char* buf = malloc(sizeof(STableBlockInfo) * pTableCheck->numOfBlocks); + char* buf = taosMemoryMalloc(sizeof(STableBlockInfo) * pTableCheck->numOfBlocks); if (buf == NULL) { cleanBlockOrderSupporter(&sup, numOfQualTables); return TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -2194,7 +2128,7 @@ static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t nu tsdbDebug("%p %d data blocks sort completed, %s", pTsdbReadHandle, cnt, pTsdbReadHandle->idStr); cleanBlockOrderSupporter(&sup, numOfTables); - free(pTree); + taosMemoryFree(pTree); return TSDB_CODE_SUCCESS; } @@ -2602,12 +2536,12 @@ static void destroyHelper(void* param) { // tQueryInfo* pInfo = (tQueryInfo*)param; // if (pInfo->optr != TSDB_RELATION_IN) { -// tfree(pInfo->q); +// taosMemoryFreeClear(pInfo->q); // } else { // taosHashCleanup((SHashObj *)(pInfo->q)); // } - free(param); + taosMemoryFree(param); } #define TSDB_PREV_ROW 0x1 @@ -2682,7 +2616,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { // return false; // } mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, 0, pRow, NULL, numOfCols, pCheckInfo->tableId, NULL, NULL, true); - tfree(pRow); + taosMemoryFreeClear(pRow); // update the last key value pCheckInfo->lastKey = key + step; @@ -2970,7 +2904,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) { // // SColumnInfoData colInfo = {{0}, 0}; // colInfo.info = pCol->info; -// colInfo.pData = calloc(1, pCol->info.bytes); +// colInfo.pData = taosMemoryCalloc(1, pCol->info.bytes); // if (colInfo.pData == NULL) { // terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; // goto out_of_memory; @@ -2989,7 +2923,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) { // cond.twindow = (STimeWindow){pTsdbReadHandle->window.skey, INT64_MAX}; // } // -// cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); +// cond.colList = taosMemoryCalloc(cond.numOfCols, sizeof(SColumnInfo)); // if (cond.colList == NULL) { // terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; // goto out_of_memory; @@ -3001,7 +2935,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) { // } // // pSecQueryHandle = tsdbQueryTablesImpl(pTsdbReadHandle->pTsdb, &cond, pTsdbReadHandle->idStr, pMemRef); -// tfree(cond.colList); +// taosMemoryFreeClear(cond.colList); // // // current table, only one table // STableCheckInfo* pCurrent = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex); @@ -3360,7 +3294,7 @@ void filterPrepare(void* expr, void* param) { return; } - pExpr->_node.info = calloc(1, sizeof(tQueryInfo)); + pExpr->_node.info = taosMemoryCalloc(1, sizeof(tQueryInfo)); STSchema* pTSSchema = (STSchema*) param; tQueryInfo* pInfo = pExpr->_node.info; @@ -3393,7 +3327,7 @@ void filterPrepare(void* expr, void* param) { size = pSchema->bytes; } // to make sure tonchar does not cause invalid write, since the '\0' needs at least sizeof(TdUcs4) space. - pInfo->q = calloc(1, size + TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); + pInfo->q = taosMemoryCalloc(1, size + TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); tVariantDump(pCond, pInfo->q, pSchema->type, true); } } @@ -3679,7 +3613,7 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch // if (tagExpr != NULL) { // CLEANUP_PUSH_VOID_PTR_PTR(true, tExprTreeDestroy, tagExpr, NULL); // tExprNode* tbnameExpr = expr; -// expr = calloc(1, sizeof(tExprNode)); +// expr = taosMemoryCalloc(1, sizeof(tExprNode)); // if (expr == NULL) { // THROW( TSDB_CODE_TDB_OUT_OF_MEMORY ); // } @@ -3793,7 +3727,7 @@ static void* doFreeColumnInfoData(SArray* pColumnInfoData) { size_t cols = taosArrayGetSize(pColumnInfoData); for (int32_t i = 0; i < cols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pColumnInfoData, i); - tfree(pColInfo->pData); + taosMemoryFreeClear(pColInfo->pData); } taosArrayDestroy(pColumnInfoData); @@ -3806,7 +3740,7 @@ static void* destroyTableCheckInfo(SArray* pTableCheckInfo) { STableCheckInfo* p = taosArrayGet(pTableCheckInfo, i); destroyTableMemIterator(p); - tfree(p->pCompInfo); + taosMemoryFreeClear(p->pCompInfo); } taosArrayDestroy(pTableCheckInfo); @@ -3823,8 +3757,8 @@ void tsdbCleanupReadHandle(tsdbReaderT queryHandle) { pTsdbReadHandle->pColumns = doFreeColumnInfoData(pTsdbReadHandle->pColumns); taosArrayDestroy(pTsdbReadHandle->defaultLoadColumn); - tfree(pTsdbReadHandle->pDataBlockInfo); - tfree(pTsdbReadHandle->statis); + taosMemoryFreeClear(pTsdbReadHandle->pDataBlockInfo); + taosMemoryFreeClear(pTsdbReadHandle->statis); if (!emptyQueryTimewindow(pTsdbReadHandle)) { // tsdbMayUnTakeMemSnapshot(pTsdbReadHandle); @@ -3849,7 +3783,7 @@ void tsdbCleanupReadHandle(tsdbReaderT queryHandle) { tsdbDebug("%p :io-cost summary: head-file read cnt:%"PRIu64", head-file time:%"PRIu64" us, statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, %s", pTsdbReadHandle, pCost->headFileLoad, pCost->headFileLoadTime, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pTsdbReadHandle->idStr); - tfree(pTsdbReadHandle); + taosMemoryFreeClear(pTsdbReadHandle); } #if 0 @@ -3908,15 +3842,15 @@ static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) { if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL || optr == TSDB_RELATION_EQUAL || optr == TSDB_RELATION_NOT_EQUAL) { - pCond->start = calloc(1, sizeof(SEndPoint)); + pCond->start = taosMemoryCalloc(1, sizeof(SEndPoint)); pCond->start->optr = queryColInfo->optr; pCond->start->v = queryColInfo->q; } else if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { - pCond->end = calloc(1, sizeof(SEndPoint)); + pCond->end = taosMemoryCalloc(1, sizeof(SEndPoint)); pCond->end->optr = queryColInfo->optr; pCond->end->v = queryColInfo->q; } else if (optr == TSDB_RELATION_IN) { - pCond->start = calloc(1, sizeof(SEndPoint)); + pCond->start = taosMemoryCalloc(1, sizeof(SEndPoint)); pCond->start->optr = queryColInfo->optr; pCond->start->v = queryColInfo->q; } else if (optr == TSDB_RELATION_LIKE) { @@ -4062,8 +3996,8 @@ static void queryIndexedColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SAr } } - free(cond.start); - free(cond.end); + taosMemoryFree(cond.start); + taosMemoryFree(cond.end); tSkipListDestroyIter(iter); } diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 0eb2d525b3fb726890ccae0befa71a9db7c35890..0873e8edc1c5f55f23244dcfb6eaff8cc96dbf57 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -25,6 +25,7 @@ static const char *TSDB_SMA_DNAME[] = { #define SMA_STORAGE_TSDB_TIMES 10 #define SMA_STORAGE_SPLIT_HOURS 24 #define SMA_KEY_LEN 18 // tableUid_colId_TSKEY 8+2+8 +#define SMA_DROP_EXPIRED_TIME 10 // default is 10 seconds #define SMA_STATE_HASH_SLOT 4 #define SMA_STATE_ITEM_HASH_SLOT 32 @@ -37,7 +38,7 @@ typedef enum { } ESmaStorageLevel; typedef struct { - STsdb * pTsdb; + STsdb *pTsdb; SDBFile dFile; int32_t interval; // interval with the precision of DB } STSmaWriteH; @@ -48,7 +49,7 @@ typedef struct { } SmaFsIter; typedef struct { - STsdb * pTsdb; + STsdb *pTsdb; SDBFile dFile; int32_t interval; // interval with the precision of DB int32_t blockSize; // size of SMA block item @@ -60,14 +61,15 @@ typedef struct { typedef struct { /** * @brief The field 'state' is here to demonstrate if one smaIndex is ready to provide service. - * - TSDB_SMA_STAT_EXPIRED: 1) If sma calculation of history TS data is not finished; 2) Or if the TSDB is open, - * without information about its previous state. * - TSDB_SMA_STAT_OK: 1) The sma calculation of history data is finished; 2) Or recevied information from * Streaming Module or TSDB local persistence. + * - TSDB_SMA_STAT_EXPIRED: 1) If sma calculation of history TS data is not finished; 2) Or if the TSDB is open, + * without information about its previous state. + * - TSDB_SMA_STAT_DROPPED: 1)sma dropped */ int8_t state; // ETsdbSmaStat SHashObj *expiredWindows; // key: skey of time window, value: N/A - STSma * pSma; // cache schema + STSma *pSma; // cache schema } SSmaStatItem; struct SSmaStat { @@ -78,8 +80,9 @@ struct SSmaStat { // declaration of static functions // expired window -static int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, ETsdbSmaType smaType, char *msg); +static int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, const char *msg); static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat); +static void *tsdbFreeSmaStatItem(SSmaStatItem *pSmaStatItem); static int32_t tsdbDestroySmaState(SSmaStat *pSmaStat); static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did); static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SDiskID did, SSmaEnv **pEnv); @@ -109,7 +112,55 @@ static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]) static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg); static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg); +// mgmt interface +static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid); + // implementation +static FORCE_INLINE int8_t tsdbSmaStat(SSmaStatItem *pStatItem) { + if (pStatItem) { + return atomic_load_8(&pStatItem->state); + } + return TSDB_SMA_STAT_UNKNOWN; +} + +static FORCE_INLINE bool tsdbSmaStatIsOK(SSmaStatItem *pStatItem, int8_t *state) { + if (!pStatItem) { + return false; + } + + if (state) { + *state = atomic_load_8(&pStatItem->state); + return *state == TSDB_SMA_STAT_OK; + } + return atomic_load_8(&pStatItem->state) == TSDB_SMA_STAT_OK; +} + +static FORCE_INLINE bool tsdbSmaStatIsExpired(SSmaStatItem *pStatItem) { + return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_EXPIRED) : true; +} + +static FORCE_INLINE bool tsdbSmaStatIsDropped(SSmaStatItem *pStatItem) { + return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_DROPPED) : true; +} + +static FORCE_INLINE void tsdbSmaStatSetOK(SSmaStatItem *pStatItem) { + if (pStatItem) { + atomic_store_8(&pStatItem->state, TSDB_SMA_STAT_OK); + } +} + +static FORCE_INLINE void tsdbSmaStatSetExpired(SSmaStatItem *pStatItem) { + if (pStatItem) { + atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_EXPIRED); + } +} + +static FORCE_INLINE void tsdbSmaStatSetDropped(SSmaStatItem *pStatItem) { + if (pStatItem) { + atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_DROPPED); + } +} + static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]) { snprintf(dirName, TSDB_FILENAME_LEN, "vnode%svnode%d%stsdb%s%s", TD_DIRSEP, vgId, TD_DIRSEP, TD_DIRSEP, TSDB_SMA_DNAME[smaType]); @@ -118,7 +169,7 @@ static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]) { static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did) { SSmaEnv *pEnv = NULL; - pEnv = (SSmaEnv *)calloc(1, sizeof(SSmaEnv)); + pEnv = (SSmaEnv *)taosMemoryCalloc(1, sizeof(SSmaEnv)); if (pEnv == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -127,7 +178,7 @@ static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did) int code = taosThreadRwlockInit(&(pEnv->lock), NULL); if (code) { terrno = TAOS_SYSTEM_ERROR(code); - free(pEnv); + taosMemoryFree(pEnv); return NULL; } @@ -179,8 +230,8 @@ static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SDiskID did, SSmaE void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv) { if (pSmaEnv) { tsdbDestroySmaState(pSmaEnv->pStat); - tfree(pSmaEnv->pStat); - tfree(pSmaEnv->path); + taosMemoryFreeClear(pSmaEnv->pStat); + taosMemoryFreeClear(pSmaEnv->path); taosThreadRwlockDestroy(&(pSmaEnv->lock)); tsdbCloseBDBEnv(pSmaEnv->dbEnv); } @@ -188,7 +239,7 @@ void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv) { void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv) { tsdbDestroySmaEnv(pSmaEnv); - tfree(pSmaEnv); + taosMemoryFreeClear(pSmaEnv); return NULL; } @@ -220,7 +271,7 @@ static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat) { * tsdbInitSmaStat invoked in other multithread environment later. */ if (*pSmaStat == NULL) { - *pSmaStat = (SSmaStat *)calloc(1, sizeof(SSmaStat)); + *pSmaStat = (SSmaStat *)taosMemoryCalloc(1, sizeof(SSmaStat)); if (*pSmaStat == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_FAILED; @@ -230,7 +281,7 @@ static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat) { taosHashInit(SMA_STATE_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if ((*pSmaStat)->smaStatItems == NULL) { - tfree(*pSmaStat); + taosMemoryFreeClear(*pSmaStat); return TSDB_CODE_FAILED; } } @@ -240,18 +291,28 @@ static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat) { static SSmaStatItem *tsdbNewSmaStatItem(int8_t state) { SSmaStatItem *pItem = NULL; - pItem = (SSmaStatItem *)calloc(1, sizeof(SSmaStatItem)); + pItem = (SSmaStatItem *)taosMemoryCalloc(1, sizeof(SSmaStatItem)); if (pItem) { pItem->state = state; pItem->expiredWindows = taosHashInit(SMA_STATE_ITEM_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP), true, HASH_ENTRY_LOCK); if (!pItem->expiredWindows) { - tfree(pItem); + taosMemoryFreeClear(pItem); } } return pItem; } +static void *tsdbFreeSmaStatItem(SSmaStatItem *pSmaStatItem) { + if (pSmaStatItem != NULL) { + tdDestroyTSma(pSmaStatItem->pSma); + taosMemoryFreeClear(pSmaStatItem->pSma); + taosHashCleanup(pSmaStatItem->expiredWindows); + taosMemoryFreeClear(pSmaStatItem); + } + return NULL; +} + /** * @brief Release resources allocated for its member fields, not including itself. * @@ -264,12 +325,7 @@ int32_t tsdbDestroySmaState(SSmaStat *pSmaStat) { void *item = taosHashIterate(pSmaStat->smaStatItems, NULL); while (item != NULL) { SSmaStatItem *pItem = *(SSmaStatItem **)item; - if (pItem != NULL) { - tdDestroyTSma(pItem->pSma); - tfree(pItem->pSma); - taosHashCleanup(pItem->expiredWindows); - tfree(pItem); - } + tsdbFreeSmaStatItem(pItem); item = taosHashIterate(pSmaStat->smaStatItems, item); } taosHashCleanup(pSmaStat->smaStatItems); @@ -328,21 +384,80 @@ static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) { return TSDB_CODE_SUCCESS; }; +static STimeWindow getActiveTimeWindowX(int64_t ts, SInterval* pInterval) { + STimeWindow tw = {0}; + tw.skey = 100; + tw.ekey = 1000; + return tw; +} + +static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey) { + SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); + if (pItem == NULL) { + pItem = tsdbNewSmaStatItem(TSDB_SMA_STAT_EXPIRED); // TODO use the real state + if (pItem == NULL) { + // Response to stream computing: OOM + // For query, if the indexUid not found, the TSDB should tell query module to query raw TS data. + return TSDB_CODE_FAILED; + } + + // cache smaMeta + STSma *pSma = metaGetSmaInfoByIndex(pTsdb->pMeta, indexUid); + if (pSma == NULL) { + terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META; + taosHashCleanup(pItem->expiredWindows); + taosMemoryFree(pItem); + tsdbWarn("vgId:%d update expired window failed for smaIndex %" PRIi64 " since %s", REPO_ID(pTsdb), indexUid, + tstrerror(terrno)); + return TSDB_CODE_FAILED; + } + pItem->pSma = pSma; + + if (taosHashPut(pItemsHash, &indexUid, sizeof(indexUid), &pItem, sizeof(pItem)) != 0) { + // If error occurs during put smaStatItem, free the resources of pItem + taosHashCleanup(pItem->expiredWindows); + taosMemoryFree(pItem); + return TSDB_CODE_FAILED; + } + } + + int8_t state = TSDB_SMA_STAT_EXPIRED; + if (taosHashPut(pItem->expiredWindows, &winSKey, sizeof(TSKEY), &state, sizeof(state)) != 0) { + // If error occurs during taosHashPut expired windows, remove the smaIndex from pTsdb->pSmaStat, thus TSDB would + // tell query module to query raw TS data. + // N.B. + // 1) It is assumed to be extemely little probability event of fail to taosHashPut. + // 2) This would solve the inconsistency to some extent, but not completely, unless we record all expired + // windows failed to put into hash table. + taosHashCleanup(pItem->expiredWindows); + taosMemoryFreeClear(pItem->pSma); + taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid)); + return TSDB_CODE_FAILED; + } + tsdbDebug("vgId:%d smaIndex %" PRIi64 " tsKey %" PRIi64 " is put to hash", REPO_ID(pTsdb), indexUid, winSKey); +} + /** * @brief Update expired window according to msg from stream computing module. * * @param pTsdb - * @param smaType ETsdbSmaType - * @param msg + * @param msg SSubmitReq * @return int32_t */ -int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, ETsdbSmaType smaType, char *msg) { - if (!msg || !pTsdb->pMeta) { +int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, const char *msg) { + const SSubmitReq *pMsg = (const SSubmitReq *)msg; + + if (pMsg->length <= sizeof(SSubmitReq)) { + terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; + return TSDB_CODE_FAILED; + } + if (!pTsdb->pMeta) { terrno = TSDB_CODE_INVALID_PTR; return TSDB_CODE_FAILED; } - // TODO: decode the msg from Stream Computing module => start +// TODO: decode the msg from Stream Computing module => start +#ifdef TSDB_SMA_TESTx int64_t indexUid = SMA_TEST_INDEX_UID; const int32_t SMA_TEST_EXPIRED_WINDOW_SIZE = 10; TSKEY expiredWindows[SMA_TEST_EXPIRED_WINDOW_SIZE]; @@ -350,75 +465,102 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, ETsdbSmaType smaType, char *msg) { for (int32_t i = 0; i < SMA_TEST_EXPIRED_WINDOW_SIZE; ++i) { expiredWindows[i] = skey1 + i; } +#else + +#endif // TODO: decode the msg <= end - if (tsdbCheckAndInitSmaEnv(pTsdb, smaType) != TSDB_CODE_SUCCESS) { + if (tsdbCheckAndInitSmaEnv(pTsdb, TSDB_SMA_TYPE_TIME_RANGE) != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TDB_INIT_FAILED; return TSDB_CODE_FAILED; } - SSmaEnv * pEnv = REPO_SMA_ENV(pTsdb, smaType); +#ifndef TSDB_SMA_TEST + TSKEY expiredWindows[SMA_TEST_EXPIRED_WINDOW_SIZE]; +#endif + + + // Firstly, assume that tSma can only be created on super table/normal table. + // getActiveTimeWindow + + + SSmaEnv *pEnv = REPO_SMA_ENV(pTsdb, TSDB_SMA_TYPE_TIME_RANGE); SSmaStat *pStat = SMA_ENV_STAT(pEnv); SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv); TASSERT(pEnv != NULL && pStat != NULL && pItemsHash != NULL); - tsdbRefSmaStat(pTsdb, pStat); - SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); - if (pItem == NULL) { - pItem = tsdbNewSmaStatItem(TSDB_SMA_STAT_EXPIRED); // TODO use the real state - if (pItem == NULL) { - // Response to stream computing: OOM - // For query, if the indexUid not found, the TSDB should tell query module to query raw TS data. - tsdbUnRefSmaStat(pTsdb, pStat); - return TSDB_CODE_FAILED; - } - // cache smaMeta - STSma *pSma = metaGetSmaInfoByIndex(pTsdb->pMeta, indexUid); - if (pSma == NULL) { - terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META; - taosHashCleanup(pItem->expiredWindows); - free(pItem); - tsdbUnRefSmaStat(pTsdb, pStat); - tsdbWarn("vgId:%d update expired window failed for smaIndex %" PRIi64 " since %s", REPO_ID(pTsdb), indexUid, - tstrerror(terrno)); - return TSDB_CODE_FAILED; - } - pItem->pSma = pSma; - if (taosHashPut(pItemsHash, &indexUid, sizeof(indexUid), &pItem, sizeof(pItem)) != 0) { - // If error occurs during put smaStatItem, free the resources of pItem - taosHashCleanup(pItem->expiredWindows); - free(pItem); - tsdbUnRefSmaStat(pTsdb, pStat); - return TSDB_CODE_FAILED; - } + SSubmitMsgIter msgIter = {0}; + SSubmitBlk *pBlock = NULL; + SInterval interval = {0}; + + + if (tInitSubmitMsgIter(pMsg, &msgIter) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_FAILED; } - int8_t state = TSDB_SMA_STAT_EXPIRED; - for (int32_t i = 0; i < SMA_TEST_EXPIRED_WINDOW_SIZE; ++i) { - if (taosHashPut(pItem->expiredWindows, expiredWindows + i, sizeof(TSKEY), &state, sizeof(state)) != 0) { - // If error occurs during taosHashPut expired windows, remove the smaIndex from pTsdb->pSmaStat, thus TSDB would - // tell query module to query raw TS data. - // N.B. - // 1) It is assumed to be extemely little probability event of fail to taosHashPut. - // 2) This would solve the inconsistency to some extent, but not completely, unless we record all expired - // windows failed to put into hash table. - taosHashCleanup(pItem->expiredWindows); - tfree(pItem->pSma); - taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid)); - tsdbUnRefSmaStat(pTsdb, pStat); - return TSDB_CODE_FAILED; + // basic procedure + // TODO: optimization + tsdbRefSmaStat(pTsdb, pStat); + + while (true) { + tGetSubmitMsgNext(&msgIter, &pBlock); + if (pBlock == NULL) break; + + int64_t suid = htobe64(pBlock->uid); + STSmaWrapper *pSW = NULL; + STSma *pTSma = NULL; + + while (true) { + SSubmitBlkIter blkIter = {0}; + if (tInitSubmitBlkIter(pBlock, &blkIter) != TSDB_CODE_SUCCESS) { + tdFreeTSmaWrapper(pSW); + break; + } + STSRow *row = tGetSubmitBlkNext(&blkIter); + if (row == NULL) { + tdFreeTSmaWrapper(pSW); + break; + } + if(pSW == NULL) { + if((pSW =metaGetSmaInfoByTable(REPO_META(pTsdb), suid)) == NULL) { + break; + } + if((pSW->number) <= 0 || (pSW->tSma == NULL)) { + tdFreeTSmaWrapper(pSW); + break; + } + pTSma = pSW->tSma; + } + + interval.interval = pTSma->interval; + interval.intervalUnit = pTSma->intervalUnit; + interval.offset = pTSma->offset; + interval.precision = REPO_CFG(pTsdb)->precision; + interval.sliding = pTSma->sliding; + interval.slidingUnit = pTSma->slidingUnit; + + STimeWindow tw = getActiveTimeWindowX(TD_ROW_KEY(row), &interval); + tsdbSetExpiredWindow(pTsdb, pItemsHash, pTSma->indexUid, TD_ROW_KEY(row)); } - tsdbDebug("vgId:%d smaIndex %" PRIi64 " tsKey %" PRIi64 " is put to hash", REPO_ID(pTsdb), indexUid, - expiredWindows[i]); } tsdbUnRefSmaStat(pTsdb, pStat); + return TSDB_CODE_SUCCESS; } +/** + * @brief When sma data received from stream computing, make the relative expired window valid. + * + * @param pTsdb + * @param pStat + * @param indexUid + * @param skey + * @return int32_t + */ static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, SSmaStat *pStat, int64_t indexUid, TSKEY skey) { SSmaStatItem *pItem = NULL; @@ -437,6 +579,15 @@ static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, SSmaStat *pStat, int64_t ind skey, indexUid); return TSDB_CODE_FAILED; } + // TODO: use a standalone interface to received state upate notification from stream computing module. + /** + * @brief state + * - When SMA env init in TSDB, its status is TSDB_SMA_STAT_OK. + * - In startup phase of stream computing module, it should notify the SMA env in TSDB to expired if needed(e.g. + * when batch data caculation not finised) + * - When TSDB_SMA_STAT_OK, the stream computing module should also notify that to the SMA env in TSDB. + */ + pItem->state = TSDB_SMA_STAT_OK; } else { // error handling tsdbUnRefSmaStat(pTsdb, pStat); @@ -517,7 +668,7 @@ static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t k #ifdef _TEST_SMA_PRINT_DEBUG_LOG_ uint32_t valueSize = 0; - void * data = tsdbGetSmaDataByKey(pDBFile, smaKey, keyLen, &valueSize); + void *data = tsdbGetSmaDataByKey(pDBFile, smaKey, keyLen, &valueSize); ASSERT(data != NULL); for (uint32_t v = 0; v < valueSize; v += 8) { tsdbWarn("vgId:%d insert sma data val[%d] %" PRIi64, REPO_ID(pSmaH->pTsdb), v, *(int64_t *)POINTER_SHIFT(data, v)); @@ -634,7 +785,7 @@ static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *p assert(pTbData->dataLen > 0); STSmaColData *pColData = (STSmaColData *)POINTER_SHIFT(pTbData->data, tbLen); char smaKey[SMA_KEY_LEN] = {0}; - void * pSmaKey = &smaKey; + void *pSmaKey = &smaKey; #if 0 printf("tsdbInsertTSmaDataSection: index %" PRIi64 ", skey %" PRIi64 " table[%" PRIi64 "]col[%" PRIu16 "]\n", pData->indexUid, pData->skey, pTbData->tableUid, pColData->colId); @@ -708,9 +859,10 @@ static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLe * @return int32_t */ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { - STsdbCfg * pCfg = REPO_CFG(pTsdb); + STsdbCfg *pCfg = REPO_CFG(pTsdb); STSmaDataWrapper *pData = (STSmaDataWrapper *)msg; - SSmaEnv * pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv); + SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv); + int64_t indexUid = SMA_TEST_INDEX_UID; if (pEnv == NULL) { terrno = TSDB_CODE_INVALID_PTR; @@ -730,13 +882,28 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { return TSDB_CODE_FAILED; } - int64_t indexUid = SMA_TEST_INDEX_UID; - char rPath[TSDB_FILENAME_LEN] = {0}; - char aPath[TSDB_FILENAME_LEN] = {0}; + SSmaStat *pStat = SMA_ENV_STAT(pTsdb->pTSmaEnv); + SSmaStatItem *pItem = NULL; + + tsdbRefSmaStat(pTsdb, pStat); + + if (pStat && pStat->smaStatItems) { + pItem = taosHashGet(pStat->smaStatItems, &indexUid, sizeof(indexUid)); + } + + if ((pItem == NULL) || ((pItem = *(SSmaStatItem **)pItem) == NULL) || tsdbSmaStatIsDropped(pItem)) { + terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + tsdbUnRefSmaStat(pTsdb, pStat); + return TSDB_CODE_FAILED; + } + + char rPath[TSDB_FILENAME_LEN] = {0}; + char aPath[TSDB_FILENAME_LEN] = {0}; snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid); tfsAbsoluteName(REPO_TFS(pTsdb), SMA_ENV_DID(pEnv), rPath, aPath); if (!taosCheckExistFile(aPath)) { if (tfsMkdirRecurAt(REPO_TFS(pTsdb), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) { + tsdbUnRefSmaStat(pTsdb, pStat); return TSDB_CODE_FAILED; } } @@ -754,12 +921,14 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { tsdbWarn("vgId:%d open DB file %s failed since %s", REPO_ID(pTsdb), tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); tsdbDestroyTSmaWriteH(&tSmaH); + tsdbUnRefSmaStat(pTsdb, pStat); return TSDB_CODE_FAILED; } if (tsdbInsertTSmaDataSection(&tSmaH, pData) != 0) { tsdbWarn("vgId:%d insert tSma data section failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); tsdbDestroyTSmaWriteH(&tSmaH); + tsdbUnRefSmaStat(pTsdb, pStat); return TSDB_CODE_FAILED; } // TODO:tsdbEndTSmaCommit(); @@ -768,9 +937,59 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { tsdbResetExpiredWindow(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv), pData->indexUid, pData->skey); tsdbDestroyTSmaWriteH(&tSmaH); + tsdbUnRefSmaStat(pTsdb, pStat); return TSDB_CODE_SUCCESS; } +/** + * @brief Drop tSma data and local cache + * - insert/query reference + * @param pTsdb + * @param msg + * @return int32_t + */ +static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid) { + SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv); + + // clear local cache + if (pEnv) { + tsdbDebug("vgId:%d drop tSma local cache for %" PRIi64, REPO_ID(pTsdb), indexUid); + + SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); + if ((pItem != NULL) || ((pItem = *(SSmaStatItem **)pItem) != NULL)) { + if (tsdbSmaStatIsDropped(pItem)) { + tsdbDebug("vgId:%d tSma stat is already dropped for %" PRIi64, REPO_ID(pTsdb), indexUid); + return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode + } + + tsdbWLockSma(pEnv); + if (tsdbSmaStatIsDropped(pItem)) { + tsdbUnLockSma(pEnv); + tsdbDebug("vgId:%d tSma stat is already dropped for %" PRIi64, REPO_ID(pTsdb), indexUid); + return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode + } + tsdbSmaStatSetDropped(pItem); + tsdbUnLockSma(pEnv); + + int32_t nSleep = 0; + while (true) { + if (T_REF_VAL_GET(SMA_ENV_STAT(pEnv)) <= 0) { + break; + } + taosSsleep(1); + if (++nSleep > SMA_DROP_EXPIRED_TIME) { + break; + }; + } + + tsdbFreeSmaStatItem(pItem); + tsdbDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64 " in local cache", REPO_ID(pTsdb), indexUid); + } + } + // clear sma data files + // TODO: +} + static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t fid) { STsdb *pTsdb = pSmaH->pTsdb; @@ -782,37 +1001,66 @@ static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, } static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) { - STsdbCfg * pCfg = REPO_CFG(pTsdb); + STsdbCfg *pCfg = REPO_CFG(pTsdb); STSmaDataWrapper *pData = (STSmaDataWrapper *)msg; - STSmaWriteH tSmaH = {0}; + SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pRSmaEnv); - tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData); + if (pEnv == NULL) { + terrno = TSDB_CODE_INVALID_PTR; + tsdbWarn("vgId:%d insert tSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); + return terrno; + } if (pData->dataLen <= 0) { TASSERT(0); terrno = TSDB_CODE_INVALID_PARA; - return terrno; + return TSDB_CODE_FAILED; } - // Step 1: Judge the storage level - int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit); - int32_t daysPerFile = storageLevel == SMA_STORAGE_LEVEL_TSDB ? SMA_STORAGE_TSDB_DAYS : pCfg->daysPerFile; + STSmaWriteH tSmaH = {0}; - // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file - // - Set and open the DFile or the B+Tree file + if (tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData) != 0) { + return TSDB_CODE_FAILED; + } + + int64_t indexUid = SMA_TEST_INDEX_UID; + char rPath[TSDB_FILENAME_LEN] = {0}; + char aPath[TSDB_FILENAME_LEN] = {0}; + snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid); + tfsAbsoluteName(REPO_TFS(pTsdb), SMA_ENV_DID(pEnv), rPath, aPath); + if (!taosCheckExistFile(aPath)) { + if (tfsMkdirRecurAt(REPO_TFS(pTsdb), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_FAILED; + } + } + // Step 1: Judge the storage level and days + int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit); + int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel); int32_t fid = (int32_t)(TSDB_KEY_FID(pData->skey, daysPerFile, pCfg->precision)); - // Save all the TSma data to one file + // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file + // - Set and open the DFile or the B+Tree file // TODO: tsdbStartTSmaCommit(); - tsdbSetTSmaDataFile(&tSmaH, pData, storageLevel, fid); + tsdbSetTSmaDataFile(&tSmaH, pData, indexUid, fid); + if (tsdbOpenDBF(pTsdb->pTSmaEnv->dbEnv, &tSmaH.dFile) != 0) { + tsdbWarn("vgId:%d open DB file %s failed since %s", REPO_ID(pTsdb), + tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); + tsdbDestroyTSmaWriteH(&tSmaH); + return TSDB_CODE_FAILED; + } - tsdbInsertTSmaDataSection(&tSmaH, pData); + if (tsdbInsertTSmaDataSection(&tSmaH, pData) != 0) { + tsdbWarn("vgId:%d insert tSma data section failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); + tsdbDestroyTSmaWriteH(&tSmaH); + return TSDB_CODE_FAILED; + } // TODO:tsdbEndTSmaCommit(); - // reset the SSmaStat - tsdbResetExpiredWindow(pTsdb, SMA_ENV_STAT(pTsdb->pRSmaEnv), pData->indexUid, pData->skey); + // Step 3: reset the SSmaStat + tsdbResetExpiredWindow(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv), pData->indexUid, pData->skey); + tsdbDestroyTSmaWriteH(&tSmaH); return TSDB_CODE_SUCCESS; } @@ -934,6 +1182,15 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_ #endif #if 1 + int8_t smaStat = 0; + if (!tsdbSmaStatIsOK(pItem, &smaStat)) { // TODO: multiple check for large scale sma query + tsdbUnRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv)); + terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + tsdbWarn("vgId:%d getTSmaDataImpl failed from index %" PRIi64 " since %s %" PRIi8, REPO_ID(pTsdb), indexUid, + tstrerror(terrno), smaStat); + return TSDB_CODE_FAILED; + } + if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY)) != NULL) { // TODO: mark this window as expired. tsdbDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, REPO_ID(pTsdb), @@ -964,7 +1221,7 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_ tReadH.dFile.path, *(tb_uid_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8), *(int64_t *)POINTER_SHIFT(smaKey, 10), SMA_KEY_LEN); - void * result = NULL; + void *result = NULL; uint32_t valueSize = 0; if ((result = tsdbGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize)) == NULL) { tsdbWarn("vgId:%d get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 @@ -980,7 +1237,7 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_ tsdbWarn("vgId:%d get sma data v[%d]=%" PRIi64, REPO_ID(pTsdb), v, *(int64_t *)POINTER_SHIFT(result, v)); } #endif - tfree(result); // TODO: fill the result to output + taosMemoryFreeClear(result); // TODO: fill the result to output #if 0 int32_t nResult = 0; @@ -1046,15 +1303,8 @@ int32_t tsdbRemoveTSmaData(STsdb *pTsdb, void *smaIndex, STimeWindow *pWin) { } #endif -/** - * @brief Insert/Update tSma(Time-range-wise SMA) data from stream computing engine - * - * @param pTsdb - * @param param - * @param msg - * @return int32_t - * TODO: Who is responsible for resource allocate and release? - */ + +// TODO: Who is responsible for resource allocate and release? int32_t tsdbInsertTSmaData(STsdb *pTsdb, char *msg) { int32_t code = TSDB_CODE_SUCCESS; if ((code = tsdbInsertTSmaDataImpl(pTsdb, msg)) < 0) { @@ -1063,21 +1313,14 @@ int32_t tsdbInsertTSmaData(STsdb *pTsdb, char *msg) { return code; } -int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, int8_t smaType, char *msg) { +int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, const char *msg) { int32_t code = TSDB_CODE_SUCCESS; - if ((code = tsdbUpdateExpiredWindow(pTsdb, smaType, msg)) < 0) { + if ((code = tsdbUpdateExpiredWindowImpl(pTsdb, msg)) < 0) { tsdbWarn("vgId:%d update expired sma window failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); } return code; } -/** - * @brief Insert Time-range-wise Rollup Sma(RSma) data - * - * @param pTsdb - * @param msg - * @return int32_t - */ int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg) { int32_t code = TSDB_CODE_SUCCESS; if ((code = tsdbInsertRSmaDataImpl(pTsdb, msg)) < 0) { @@ -1086,6 +1329,7 @@ int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg) { return code; } + int32_t tsdbGetTSmaData(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySKey, int32_t nMaxResult) { int32_t code = TSDB_CODE_SUCCESS; @@ -1094,4 +1338,13 @@ int32_t tsdbGetTSmaData(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, tsdbWarn("vgId:%d get tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); } return code; +} + + +int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid) { + int32_t code = TSDB_CODE_SUCCESS; + if ((code = tsdbDropTSmaDataImpl(pTsdb, indexUid)) < 0) { + tsdbWarn("vgId:%d drop tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); + } + return code; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 3ccb483fe48c459b52dcd783ec62ee45ad9e4112..5590f13cc6e9c602a5530910de3c0154d547219f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -32,48 +32,4 @@ int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp) { } } return tsdbMemTableInsert(pTsdb, pTsdb->mem, pMsg, NULL); -} - -#if 0 -/** - * @brief Insert/Update tSma(Time-range-wise SMA) data from stream computing engine - * - * @param pTsdb - * @param param - * @param msg - * @return int32_t - * TODO: Who is responsible for resource allocate and release? - */ -int32_t tsdbInsertTSmaData(STsdb *pTsdb, char *msg) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tsdbInsertTSmaDataImpl(pTsdb, msg)) < 0) { - tsdbWarn("vgId:%d insert tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); - } - return code; -} - -int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, int8_t smaType, char *msg) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tsdbUpdateExpiredWindow(pTsdb, smaType, msg)) < 0) { - tsdbWarn("vgId:%d update expired sma window failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); - } - return code; -} - -/** - * @brief Insert Time-range-wise Rollup Sma(RSma) data - * - * @param pTsdb - * @param param - * @param msg - * @return int32_t - */ -int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tsdbInsertRSmaDataImpl(pTsdb, msg)) < 0) { - tsdbWarn("vgId:%d insert rSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); - } - return code; -} - -#endif \ No newline at end of file +} \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c b/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c index 14b9a5124ffcb89f91194190b661f5e70062c98f..ab0da1451dec1b818290cce00b21d4df62b1af2d 100644 --- a/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c +++ b/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c @@ -19,7 +19,7 @@ static SVArenaNode *vArenaNodeNew(uint64_t capacity); static void vArenaNodeFree(SVArenaNode *pNode); SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { - SVMemAllocator *pVMA = (SVMemAllocator *)malloc(sizeof(*pVMA)); + SVMemAllocator *pVMA = (SVMemAllocator *)taosMemoryMalloc(sizeof(*pVMA)); if (pVMA == NULL) { return NULL; } @@ -31,7 +31,7 @@ SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { pVMA->pNode = vArenaNodeNew(capacity); if (pVMA->pNode == NULL) { - free(pVMA); + taosMemoryFree(pVMA); return NULL; } @@ -48,7 +48,7 @@ void vmaDestroy(SVMemAllocator *pVMA) { vArenaNodeFree(pNode); } - free(pVMA); + taosMemoryFree(pVMA); } } @@ -99,7 +99,7 @@ bool vmaIsFull(SVMemAllocator *pVMA) { static SVArenaNode *vArenaNodeNew(uint64_t capacity) { SVArenaNode *pNode = NULL; - pNode = (SVArenaNode *)malloc(sizeof(*pNode) + capacity); + pNode = (SVArenaNode *)taosMemoryMalloc(sizeof(*pNode) + capacity); if (pNode == NULL) { return NULL; } @@ -112,6 +112,6 @@ static SVArenaNode *vArenaNodeNew(uint64_t capacity) { static void vArenaNodeFree(SVArenaNode *pNode) { if (pNode) { - free(pNode); + taosMemoryFree(pNode); } } diff --git a/source/dnode/vnode/src/vnd/vnodeBufferPool.c b/source/dnode/vnode/src/vnd/vnodeBufferPool.c index 3b43fd82f6973f252a833fb03eb54ff201bb55bd..5be88cdc2e0fc2b9871a55be8e735e06ba1ddf2f 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufferPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufferPool.c @@ -34,7 +34,7 @@ static void vBufPoolDestroyMA(SMemAllocatorFactory *pMAF, SMemAllocato int vnodeOpenBufPool(SVnode *pVnode) { uint64_t capacity; - if ((pVnode->pBufPool = (SVBufPool *)calloc(1, sizeof(SVBufPool))) == NULL) { + if ((pVnode->pBufPool = (SVBufPool *)taosMemoryCalloc(1, sizeof(SVBufPool))) == NULL) { /* TODO */ return -1; } @@ -57,7 +57,7 @@ int vnodeOpenBufPool(SVnode *pVnode) { TD_DLIST_APPEND(&(pVnode->pBufPool->free), pVMA); } - pVnode->pBufPool->pMAF = (SMemAllocatorFactory *)malloc(sizeof(SMemAllocatorFactory)); + pVnode->pBufPool->pMAF = (SMemAllocatorFactory *)taosMemoryMalloc(sizeof(SMemAllocatorFactory)); if (pVnode->pBufPool->pMAF == NULL) { // TODO: handle error return -1; @@ -71,7 +71,7 @@ int vnodeOpenBufPool(SVnode *pVnode) { void vnodeCloseBufPool(SVnode *pVnode) { if (pVnode->pBufPool) { - tfree(pVnode->pBufPool->pMAF); + taosMemoryFreeClear(pVnode->pBufPool->pMAF); vmaDestroy(pVnode->pBufPool->inuse); while (true) { @@ -88,7 +88,7 @@ void vnodeCloseBufPool(SVnode *pVnode) { vmaDestroy(pVMA); } - free(pVnode->pBufPool); + taosMemoryFree(pVnode->pBufPool); pVnode->pBufPool = NULL; } } @@ -161,7 +161,7 @@ static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pMAF) { SVnode * pVnode = (SVnode *)(pMAF->impl); SVMAWrapper * pWrapper; - pMA = (SMemAllocator *)calloc(1, sizeof(*pMA) + sizeof(SVMAWrapper)); + pMA = (SMemAllocator *)taosMemoryCalloc(1, sizeof(*pMA) + sizeof(SVMAWrapper)); if (pMA == NULL) { return NULL; } @@ -182,7 +182,7 @@ static void vBufPoolDestroyMA(SMemAllocatorFactory *pMAF, SMemAllocator *pMA) { SVnode * pVnode = pWrapper->pVnode; SVMemAllocator *pVMA = pWrapper->pVMA; - free(pMA); + taosMemoryFree(pMA); if (--pVMA->_ref.val == 0) { TD_DLIST_POP(&(pVnode->pBufPool->incycle), pVMA); vmaReset(pVMA); diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index bc7a8460b8b842a0d8d008c650d7d78db438d1c0..696c5f39f62b93e2b9ecbdc7bb6af6e2afcb975f 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -24,7 +24,7 @@ int vnodeAsyncCommit(SVnode *pVnode) { vnodeWaitCommit(pVnode); vnodeBufPoolSwitch(pVnode); - SVnodeTask *pTask = (SVnodeTask *)malloc(sizeof(*pTask)); + SVnodeTask *pTask = (SVnodeTask *)taosMemoryMalloc(sizeof(*pTask)); pTask->execute = vnodeCommit; // TODO pTask->arg = pVnode; // TODO diff --git a/source/dnode/vnode/src/vnd/vnodeMain.c b/source/dnode/vnode/src/vnd/vnodeMain.c index 86e670d533999baab19b332ff5617982df65a2ba..91c6e4d263181ada878ba3d9cdc52771233aeb1d 100644 --- a/source/dnode/vnode/src/vnd/vnodeMain.c +++ b/source/dnode/vnode/src/vnd/vnodeMain.c @@ -72,7 +72,7 @@ void vnodeDestroy(const char *path) { taosRemoveDir(path); } static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg) { SVnode *pVnode = NULL; - pVnode = (SVnode *)calloc(1, sizeof(*pVnode)); + pVnode = (SVnode *)taosMemoryCalloc(1, sizeof(*pVnode)); if (pVnode == NULL) { // TODO return NULL; @@ -92,8 +92,8 @@ static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg) { static void vnodeFree(SVnode *pVnode) { if (pVnode) { tsem_destroy(&(pVnode->canCommit)); - tfree(pVnode->path); - free(pVnode); + taosMemoryFreeClear(pVnode->path); + taosMemoryFree(pVnode); } } @@ -115,7 +115,8 @@ static int vnodeOpenImpl(SVnode *pVnode) { // Open tsdb sprintf(dir, "%s/tsdb", pVnode->path); - pVnode->pTsdb = tsdbOpen(dir, pVnode->vgId, &(pVnode->config.tsdbCfg), vBufPoolGetMAF(pVnode), pVnode->pMeta, pVnode->pTfs); + pVnode->pTsdb = + tsdbOpen(dir, pVnode->vgId, &(pVnode->config.tsdbCfg), vBufPoolGetMAF(pVnode), pVnode->pMeta, pVnode->pTfs); if (pVnode->pTsdb == NULL) { // TODO: handle error return -1; @@ -131,7 +132,7 @@ static int vnodeOpenImpl(SVnode *pVnode) { // Open TQ sprintf(dir, "%s/tq", pVnode->path); - pVnode->pTq = tqOpen(dir, pVnode->pWal, pVnode->pMeta, &(pVnode->config.tqCfg), vBufPoolGetMAF(pVnode)); + pVnode->pTq = tqOpen(dir, pVnode, pVnode->pWal, pVnode->pMeta, &(pVnode->config.tqCfg), vBufPoolGetMAF(pVnode)); if (pVnode->pTq == NULL) { // TODO: handle error return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeMgr.c b/source/dnode/vnode/src/vnd/vnodeMgr.c index 920b5b0947f93b1aaf04b8085b0571dd14a2d0e3..8f7d5713ab69be0baceb80f779a1fbb23246e2bb 100644 --- a/source/dnode/vnode/src/vnd/vnodeMgr.c +++ b/source/dnode/vnode/src/vnd/vnodeMgr.c @@ -29,7 +29,7 @@ int vnodeInit() { // Start commit handers vnodeMgr.nthreads = tsNumOfCommitThreads; - vnodeMgr.threads = calloc(vnodeMgr.nthreads, sizeof(TdThread)); + vnodeMgr.threads = taosMemoryCalloc(vnodeMgr.nthreads, sizeof(TdThread)); if (vnodeMgr.threads == NULL) { return -1; } @@ -65,7 +65,7 @@ void vnodeCleanup() { taosThreadJoin(vnodeMgr.threads[i], NULL); } - tfree(vnodeMgr.threads); + taosMemoryFreeClear(vnodeMgr.threads); taosThreadCondDestroy(&(vnodeMgr.hasTask)); taosThreadMutexDestroy(&(vnodeMgr.mutex)); } @@ -107,7 +107,7 @@ static void* loop(void* arg) { taosThreadMutexUnlock(&(vnodeMgr.mutex)); (*(pTask->execute))(pTask->arg); - free(pTask); + taosMemoryFree(pTask); } return NULL; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 2f5f94b55f9756b8fc802f4782743e035114728e..6b5eb5bd6f90f25305390f2330ebe6519c363a89 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -68,6 +68,8 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg) { return tqProcessPollReq(pVnode->pTq, pMsg); case TDMT_VND_TASK_EXEC: return tqProcessTaskExec(pVnode->pTq, pMsg); + case TDMT_VND_STREAM_TRIGGER: + return tqProcessStreamTrigger(pVnode->pTq, pMsg->pCont, pMsg->contLen); case TDMT_VND_QUERY_HEARTBEAT: return qWorkerProcessHbMsg(pVnode, pVnode->pQuery, pMsg); default: @@ -139,7 +141,7 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { pTagSchema = NULL; } - metaRsp.pSchemas = calloc(nCols + nTagCols, sizeof(SSchema)); + metaRsp.pSchemas = taosMemoryCalloc(nCols + nTagCols, sizeof(SSchema)); if (metaRsp.pSchemas == NULL) { code = TSDB_CODE_VND_OUT_OF_MEMORY; goto _exit; @@ -163,7 +165,6 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { memcpy(POINTER_SHIFT(metaRsp.pSchemas, sizeof(SSchema) * pSW->nCols), pTagSchema, sizeof(SSchema) * nTagCols); } - _exit: rspLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp); @@ -179,22 +180,21 @@ _exit: } tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp); - tFreeSTableMetaRsp(&metaRsp); if (pSW != NULL) { - tfree(pSW->pSchema); - tfree(pSW); + taosMemoryFreeClear(pSW->pSchema); + taosMemoryFreeClear(pSW); } if (pTbCfg) { - tfree(pTbCfg->name); + taosMemoryFreeClear(pTbCfg->name); if (pTbCfg->type == META_SUPER_TABLE) { - free(pTbCfg->stbCfg.pTagSchema); + taosMemoryFree(pTbCfg->stbCfg.pTagSchema); } else if (pTbCfg->type == META_SUPER_TABLE) { kvRowFree(pTbCfg->ctbCfg.pTag); } - tfree(pTbCfg); + taosMemoryFreeClear(pTbCfg); } rpcMsg.handle = pMsg->handle; @@ -210,7 +210,7 @@ _exit: static void freeItemHelper(void *pItem) { char *p = *(char **)pItem; - free(p); + taosMemoryFree(p); } /** @@ -230,7 +230,7 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { taosArrayPush(pArray, &name); totalLen += strlen(name); } else { - tfree(name); + taosMemoryFreeClear(name); } numOfTables++; @@ -260,7 +260,7 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { STR_TO_VARSTR(p, n); p += (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE); - // free(n); + // taosMemoryFree(n); } pFetchRsp->numOfRows = htonl(numOfTables); diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index ade9adb1e13f3a566e297421f3a6df49918f7d6d..e0be9ed89a239cc29fa6f741af785fc4d8266d25 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -59,7 +59,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // todo: change the interface here int64_t ver; taosDecodeFixedI64(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &ver); - if (tqPushMsg(pVnode->pTq, ptr, pMsg->msgType, ver) < 0) { + if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, ver) < 0) { // TODO: handle error } @@ -72,9 +72,9 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } // TODO: maybe need to clear the request struct - free(vCreateTbReq.stbCfg.pSchema); - free(vCreateTbReq.stbCfg.pTagSchema); - free(vCreateTbReq.name); + taosMemoryFree(vCreateTbReq.stbCfg.pSchema); + taosMemoryFree(vCreateTbReq.stbCfg.pTagSchema); + taosMemoryFree(vCreateTbReq.name); break; } case TDMT_VND_CREATE_TABLE: { @@ -85,10 +85,10 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { for (int i = 0; i < reqNum; i++) { SVCreateTbReq *pCreateTbReq = taosArrayGet(vCreateTbBatchReq.pArray, i); - char tableFName[TSDB_TABLE_FNAME_LEN]; + char tableFName[TSDB_TABLE_FNAME_LEN]; SMsgHead *pHead = (SMsgHead *)pMsg->pCont; sprintf(tableFName, "%s.%s", pCreateTbReq->dbFName, pCreateTbReq->name); - + int32_t code = vnodeValidateTableHash(&pVnode->config, tableFName); if (code) { SVCreateTbRsp rsp; @@ -96,19 +96,19 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { taosArrayPush(vCreateTbBatchRsp.rspList, &rsp); } - + if (metaCreateTable(pVnode->pMeta, pCreateTbReq) < 0) { // TODO: handle error vError("vgId:%d, failed to create table: %s", pVnode->vgId, pCreateTbReq->name); } - free(pCreateTbReq->name); + taosMemoryFree(pCreateTbReq->name); if (pCreateTbReq->type == TD_SUPER_TABLE) { - free(pCreateTbReq->stbCfg.pSchema); - free(pCreateTbReq->stbCfg.pTagSchema); + taosMemoryFree(pCreateTbReq->stbCfg.pSchema); + taosMemoryFree(pCreateTbReq->stbCfg.pTagSchema); } else if (pCreateTbReq->type == TD_CHILD_TABLE) { - free(pCreateTbReq->ctbCfg.pTag); + taosMemoryFree(pCreateTbReq->ctbCfg.pTag); } else { - free(pCreateTbReq->ntbCfg.pSchema); + taosMemoryFree(pCreateTbReq->ntbCfg.pSchema); } } @@ -116,11 +116,11 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { taosArrayDestroy(vCreateTbBatchReq.pArray); if (vCreateTbBatchRsp.rspList) { int32_t contLen = tSerializeSVCreateTbBatchRsp(NULL, 0, &vCreateTbBatchRsp); - void *msg = rpcMallocCont(contLen); + void *msg = rpcMallocCont(contLen); tSerializeSVCreateTbBatchRsp(msg, contLen, &vCreateTbBatchRsp); taosArrayDestroy(vCreateTbBatchRsp.rspList); - - *pRsp = calloc(1, sizeof(SRpcMsg)); + + *pRsp = taosMemoryCalloc(1, sizeof(SRpcMsg)); (*pRsp)->msgType = TDMT_VND_CREATE_TABLE_RSP; (*pRsp)->pCont = msg; (*pRsp)->contLen = contLen; @@ -133,9 +133,9 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { SVCreateTbReq vAlterTbReq = {0}; vTrace("vgId:%d, process alter stb req", pVnode->vgId); tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vAlterTbReq); - free(vAlterTbReq.stbCfg.pSchema); - free(vAlterTbReq.stbCfg.pTagSchema); - free(vAlterTbReq.name); + taosMemoryFree(vAlterTbReq.stbCfg.pSchema); + taosMemoryFree(vAlterTbReq.stbCfg.pTagSchema); + taosMemoryFree(vAlterTbReq.name); break; } case TDMT_VND_DROP_STB: @@ -168,6 +168,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } } break; case TDMT_VND_CREATE_SMA: { // timeRangeSMA +#if 0 SSmaCfg vCreateSmaReq = {0}; if (tDeserializeSVCreateTSmaReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateSmaReq) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -189,26 +190,37 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // } tdDestroyTSma(&vCreateSmaReq.tSma); // TODO: return directly or go on follow steps? +#endif } break; case TDMT_VND_CANCEL_SMA: { // timeRangeSMA } break; case TDMT_VND_DROP_SMA: { // timeRangeSMA +#if 0 SVDropTSmaReq vDropSmaReq = {0}; if (tDeserializeSVDropTSmaReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vDropSmaReq) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - if (metaDropTSma(pVnode->pMeta, vDropSmaReq.indexName) < 0) { - // TODO: handle error - return -1; - } // TODO: send msg to stream computing to drop tSma // if ((send msg to stream computing) < 0) { // tdDestroyTSma(&vCreateSmaReq); // return -1; // } + // + + if (metaDropTSma(pVnode->pMeta, vDropSmaReq.indexUid) < 0) { + // TODO: handle error + return -1; + } + + if(tsdbDropTSmaData(pVnode->pTsdb, vDropSmaReq.indexUid) < 0) { + // TODO: handle error + return -1; + } + // TODO: return directly or go on follow steps? +#endif } break; default: ASSERT(0); diff --git a/source/dnode/vnode/test/tqMetaTest.cpp b/source/dnode/vnode/test/tqMetaTest.cpp index 4f1518525476127241401741c371f0a8071dc6d9..627dbc6f18122e39cde2e243fd564ad622324cd2 100644 --- a/source/dnode/vnode/test/tqMetaTest.cpp +++ b/source/dnode/vnode/test/tqMetaTest.cpp @@ -12,7 +12,7 @@ struct Foo { int FooSerializer(const void* pObj, STqSerializedHead** ppHead) { Foo* foo = (Foo*)pObj; if ((*ppHead) == NULL || (*ppHead)->ssize < sizeof(STqSerializedHead) + sizeof(int32_t)) { - *ppHead = (STqSerializedHead*)realloc(*ppHead, sizeof(STqSerializedHead) + sizeof(int32_t)); + *ppHead = (STqSerializedHead*)taosMemoryRealloc(*ppHead, sizeof(STqSerializedHead) + sizeof(int32_t)); (*ppHead)->ssize = sizeof(STqSerializedHead) + sizeof(int32_t); } *(int32_t*)(*ppHead)->content = foo->a; @@ -21,14 +21,14 @@ int FooSerializer(const void* pObj, STqSerializedHead** ppHead) { const void* FooDeserializer(const STqSerializedHead* pHead, void** ppObj) { if (*ppObj == NULL) { - *ppObj = realloc(*ppObj, sizeof(int32_t)); + *ppObj = taosMemoryRealloc(*ppObj, sizeof(int32_t)); } Foo* pFoo = *(Foo**)ppObj; pFoo->a = *(int32_t*)pHead->content; return NULL; } -void FooDeleter(void* pObj) { free(pObj); } +void FooDeleter(void* pObj) { taosMemoryFree(pObj); } class TqMetaUpdateAppendTest : public ::testing::Test { protected: @@ -58,7 +58,7 @@ TEST_F(TqMetaUpdateAppendTest, copyPutTest) { } TEST_F(TqMetaUpdateAppendTest, persistTest) { - Foo* pFoo = (Foo*)malloc(sizeof(Foo)); + Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo)); pFoo->a = 2; tqHandleMovePut(pMeta, 1, pFoo); Foo* pBar = (Foo*)tqHandleGet(pMeta, 1); @@ -82,7 +82,7 @@ TEST_F(TqMetaUpdateAppendTest, persistTest) { } TEST_F(TqMetaUpdateAppendTest, uncommittedTest) { - Foo* pFoo = (Foo*)malloc(sizeof(Foo)); + Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo)); pFoo->a = 3; tqHandleMovePut(pMeta, 1, pFoo); @@ -91,7 +91,7 @@ TEST_F(TqMetaUpdateAppendTest, uncommittedTest) { } TEST_F(TqMetaUpdateAppendTest, abortTest) { - Foo* pFoo = (Foo*)malloc(sizeof(Foo)); + Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo)); pFoo->a = 3; tqHandleMovePut(pMeta, 1, pFoo); @@ -104,7 +104,7 @@ TEST_F(TqMetaUpdateAppendTest, abortTest) { } TEST_F(TqMetaUpdateAppendTest, deleteTest) { - Foo* pFoo = (Foo*)malloc(sizeof(Foo)); + Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo)); pFoo->a = 3; tqHandleMovePut(pMeta, 1, pFoo); @@ -135,12 +135,12 @@ TEST_F(TqMetaUpdateAppendTest, deleteTest) { } TEST_F(TqMetaUpdateAppendTest, intxnPersist) { - Foo* pFoo = (Foo*)malloc(sizeof(Foo)); + Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo)); pFoo->a = 3; tqHandleMovePut(pMeta, 1, pFoo); tqHandleCommit(pMeta, 1); - Foo* pBar = (Foo*)malloc(sizeof(Foo)); + Foo* pBar = (Foo*)taosMemoryMalloc(sizeof(Foo)); pBar->a = 4; tqHandleMovePut(pMeta, 1, pBar); diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index 3c51ad5d71284fa635349a565cc971383bbb0a69..c9488549c64885e5c213586fd01d44820c8734e0 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -51,12 +51,12 @@ TEST(testCase, unionEncodeDecodeTest) { sut.type = 1; sut.nBSmaCols = 2; - sut.pBSmaCols = (col_id_t*)malloc(sut.nBSmaCols * sizeof(col_id_t)); + sut.pBSmaCols = (col_id_t*)taosMemoryMalloc(sut.nBSmaCols * sizeof(col_id_t)); for (col_id_t i = 0; i < sut.nBSmaCols; ++i) { sut.pBSmaCols[i] = i + 100; } - void* buf = malloc(1024); + void* buf = taosMemoryMalloc(1024); void * pBuf = buf; int32_t tlen = 0; tlen += taosEncodeFixedU8(&buf, sut.info); @@ -69,7 +69,7 @@ TEST(testCase, unionEncodeDecodeTest) { pBuf = taosDecodeFixedU8(pBuf, &dut.info); pBuf = taosDecodeFixedI16(pBuf, &dut.nBSmaCols); if(dut.nBSmaCols > 0) { - dut.pBSmaCols = (col_id_t*)malloc(dut.nBSmaCols * sizeof(col_id_t)); + dut.pBSmaCols = (col_id_t*)taosMemoryMalloc(dut.nBSmaCols * sizeof(col_id_t)); for(col_id_t i=0; i < dut.nBSmaCols; ++i) { pBuf = taosDecodeFixedI16(pBuf, dut.pBSmaCols + i); } @@ -98,14 +98,14 @@ TEST(testCase, tSma_Meta_Encode_Decode_Test) { tSma.slidingUnit = TIME_UNIT_HOUR; tSma.sliding = 0; tstrncpy(tSma.indexName, "sma_index_test", TSDB_INDEX_NAME_LEN); - tstrncpy(tSma.timezone, "Asia/Shanghai", TD_TIMEZONE_LEN); + tSma.timezoneInt = 8; tSma.indexUid = 2345678910; tSma.tableUid = 1234567890; STSmaWrapper tSmaWrapper = {.number = 1, .tSma = &tSma}; uint32_t bufLen = tEncodeTSmaWrapper(NULL, &tSmaWrapper); - void *buf = calloc(1, bufLen); + void *buf = taosMemoryCalloc(1, bufLen); ASSERT_NE(buf, nullptr); STSmaWrapper *pSW = (STSmaWrapper *)buf; @@ -128,7 +128,7 @@ TEST(testCase, tSma_Meta_Encode_Decode_Test) { ASSERT_EQ(pSma->intervalUnit, qSma->intervalUnit); ASSERT_EQ(pSma->slidingUnit, qSma->slidingUnit); ASSERT_STRCASEEQ(pSma->indexName, qSma->indexName); - ASSERT_STRCASEEQ(pSma->timezone, qSma->timezone); + ASSERT_EQ(pSma->timezoneInt, qSma->timezoneInt); ASSERT_EQ(pSma->indexUid, qSma->indexUid); ASSERT_EQ(pSma->tableUid, qSma->tableUid); ASSERT_EQ(pSma->interval, qSma->interval); @@ -140,7 +140,7 @@ TEST(testCase, tSma_Meta_Encode_Decode_Test) { } // resource release - tfree(pSW); + taosMemoryFreeClear(pSW); tdDestroyTSma(&tSma); tdDestroyTSmaWrapper(&dstTSmaWrapper); } @@ -150,7 +150,7 @@ TEST(testCase, tSma_Meta_Encode_Decode_Test) { TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { const char * smaIndexName1 = "sma_index_test_1"; const char * smaIndexName2 = "sma_index_test_2"; - const char * timezone = "Asia/Shanghai"; + int8_t timezone = 8; const char * expr = "select count(a,b, top 20), from table interval 1d, sliding 1h;"; const char * tagsFilter = "I'm tags filter"; const char * smaTestDir = "./smaTest"; @@ -167,16 +167,16 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { tSma.sliding = 0; tSma.indexUid = indexUid1; tstrncpy(tSma.indexName, smaIndexName1, TSDB_INDEX_NAME_LEN); - tstrncpy(tSma.timezone, timezone, TD_TIMEZONE_LEN); + tSma.timezoneInt = 8; tSma.tableUid = tbUid; tSma.exprLen = strlen(expr); - tSma.expr = (char *)calloc(1, tSma.exprLen + 1); + tSma.expr = (char *)taosMemoryCalloc(1, tSma.exprLen + 1); ASSERT_NE(tSma.expr, nullptr); tstrncpy(tSma.expr, expr, tSma.exprLen + 1); tSma.tagsFilterLen = strlen(tagsFilter); - tSma.tagsFilter = (char *)calloc(tSma.tagsFilterLen + 1, 1); + tSma.tagsFilter = (char *)taosMemoryCalloc(tSma.tagsFilterLen + 1, 1); ASSERT_NE(tSma.tagsFilter, nullptr); tstrncpy(tSma.tagsFilter, tagsFilter, tSma.tagsFilterLen + 1); @@ -207,24 +207,24 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { qSmaCfg = metaGetSmaInfoByIndex(pMeta, indexUid1); assert(qSmaCfg != NULL); printf("name1 = %s\n", qSmaCfg->indexName); - printf("timezone1 = %s\n", qSmaCfg->timezone); + printf("timezone1 = %" PRIi8 "\n", qSmaCfg->timezoneInt); printf("expr1 = %s\n", qSmaCfg->expr != NULL ? qSmaCfg->expr : ""); printf("tagsFilter1 = %s\n", qSmaCfg->tagsFilter != NULL ? qSmaCfg->tagsFilter : ""); ASSERT_STRCASEEQ(qSmaCfg->indexName, smaIndexName1); ASSERT_EQ(qSmaCfg->tableUid, tSma.tableUid); tdDestroyTSma(qSmaCfg); - tfree(qSmaCfg); + taosMemoryFreeClear(qSmaCfg); qSmaCfg = metaGetSmaInfoByIndex(pMeta, indexUid2); assert(qSmaCfg != NULL); printf("name2 = %s\n", qSmaCfg->indexName); - printf("timezone2 = %s\n", qSmaCfg->timezone); + printf("timezone2 = %" PRIi8 "\n", qSmaCfg->timezoneInt); printf("expr2 = %s\n", qSmaCfg->expr != NULL ? qSmaCfg->expr : ""); printf("tagsFilter2 = %s\n", qSmaCfg->tagsFilter != NULL ? qSmaCfg->tagsFilter : ""); ASSERT_STRCASEEQ(qSmaCfg->indexName, smaIndexName2); ASSERT_EQ(qSmaCfg->interval, tSma.interval); tdDestroyTSma(qSmaCfg); - tfree(qSmaCfg); + taosMemoryFreeClear(qSmaCfg); // get index name by table uid SMSmaCursor *pSmaCur = metaOpenSmaCursor(pMeta, tbUid); @@ -246,20 +246,20 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { assert(pSW != NULL); ASSERT_EQ(pSW->number, nCntTSma); ASSERT_STRCASEEQ(pSW->tSma->indexName, smaIndexName1); - ASSERT_STRCASEEQ(pSW->tSma->timezone, timezone); + ASSERT_EQ(pSW->tSma->timezoneInt, timezone); ASSERT_STRCASEEQ(pSW->tSma->expr, expr); ASSERT_STRCASEEQ(pSW->tSma->tagsFilter, tagsFilter); ASSERT_EQ(pSW->tSma->indexUid, indexUid1); ASSERT_EQ(pSW->tSma->tableUid, tbUid); ASSERT_STRCASEEQ((pSW->tSma + 1)->indexName, smaIndexName2); - ASSERT_STRCASEEQ((pSW->tSma + 1)->timezone, timezone); + ASSERT_EQ((pSW->tSma + 1)->timezoneInt, timezone); ASSERT_STRCASEEQ((pSW->tSma + 1)->expr, expr); ASSERT_STRCASEEQ((pSW->tSma + 1)->tagsFilter, tagsFilter); ASSERT_EQ((pSW->tSma + 1)->indexUid, indexUid2); ASSERT_EQ((pSW->tSma + 1)->tableUid, tbUid); tdDestroyTSmaWrapper(pSW); - tfree(pSW); + taosMemoryFreeClear(pSW); // get all sma table uids SArray *pUids = metaGetSmaTbUids(pMeta, false); @@ -272,8 +272,8 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { taosArrayDestroy(pUids); // resource release - metaRemoveSmaFromDb(pMeta, smaIndexName1); - metaRemoveSmaFromDb(pMeta, smaIndexName2); + metaRemoveSmaFromDb(pMeta, indexUid1); + metaRemoveSmaFromDb(pMeta, indexUid2); tdDestroyTSma(&tSma); metaClose(pMeta); @@ -284,7 +284,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { TEST(testCase, tSma_Data_Insert_Query_Test) { // step 1: prepare meta const char * smaIndexName1 = "sma_index_test_1"; - const char * timezone = "Asia/Shanghai"; + const int8_t timezone = 8; const char * expr = "select count(a,b, top 20), from table interval 1d, sliding 1h;"; const char * tagsFilter = "where tags.location='Beijing' and tags.district='ChaoYang'"; const char * smaTestDir = "./smaTest"; @@ -305,16 +305,16 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { tSma.sliding = 0; tSma.indexUid = indexUid1; tstrncpy(tSma.indexName, smaIndexName1, TSDB_INDEX_NAME_LEN); - tstrncpy(tSma.timezone, timezone, TD_TIMEZONE_LEN); + tSma.timezoneInt = timezone; tSma.tableUid = tbUid; tSma.exprLen = strlen(expr); - tSma.expr = (char *)calloc(1, tSma.exprLen + 1); + tSma.expr = (char *)taosMemoryCalloc(1, tSma.exprLen + 1); ASSERT_NE(tSma.expr, nullptr); tstrncpy(tSma.expr, expr, tSma.exprLen + 1); tSma.tagsFilterLen = strlen(tagsFilter); - tSma.tagsFilter = (char *)calloc(1, tSma.tagsFilterLen + 1); + tSma.tagsFilter = (char *)taosMemoryCalloc(1, tSma.tagsFilterLen + 1); ASSERT_NE(tSma.tagsFilter, nullptr); tstrncpy(tSma.tagsFilter, tagsFilter, tSma.tagsFilterLen + 1); @@ -331,7 +331,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { // step 2: insert data STSmaDataWrapper *pSmaData = NULL; - STsdb * pTsdb = (STsdb *)calloc(1, sizeof(STsdb)); + STsdb * pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(STsdb)); STsdbCfg * pCfg = &pTsdb->config; pTsdb->pMeta = pMeta; @@ -367,9 +367,9 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { pTsdb->pTfs = tfsOpen(&pDisks, numOfDisks); ASSERT_NE(pTsdb->pTfs, nullptr); - char *msg = (char *)calloc(1, 100); + char *msg = (char *)taosMemoryCalloc(1, 100); ASSERT_NE(msg, nullptr); - ASSERT_EQ(tsdbUpdateSmaWindow(pTsdb, TSDB_SMA_TYPE_TIME_RANGE, msg), 0); + ASSERT_EQ(tsdbUpdateSmaWindow(pTsdb, msg), 0); // init int32_t allocCnt = 0; @@ -443,7 +443,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { printf("%s:%d The sma data check count for insert and query is %" PRIu32 "\n", __FILE__, __LINE__, checkDataCnt); // release data - tfree(msg); + taosMemoryFreeClear(msg); taosTZfree(buf); // release meta tdDestroyTSma(&tSma); diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index ea8195bfd11081dd506ee61302a8141c99038b02..b448a43dcb332804e4c91de89e6fb1f2c8549770 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -8,6 +8,7 @@ add_subdirectory(scheduler) add_subdirectory(cache) add_subdirectory(catalog) add_subdirectory(executor) +add_subdirectory(stream) add_subdirectory(planner) add_subdirectory(function) add_subdirectory(qcom) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 8f674826509546b216e387c928cb9f89e3f7fb0f..7bbff05b4b37449abed5930f71f40fb8379d1fef 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -242,7 +242,7 @@ void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { } } - tfree(mgmt->slots); + taosMemoryFreeClear(mgmt->slots); } @@ -272,7 +272,7 @@ void ctgFreeVgInfo(SDBVgInfo *vgInfo) { vgInfo->vgHash = NULL; } - tfree(vgInfo); + taosMemoryFreeClear(vgInfo); } void ctgFreeDbCache(SCtgDBCache *dbCache) { @@ -307,7 +307,7 @@ void ctgFreeHandle(SCatalog* pCtg) { taosHashCleanup(pCtg->dbCache); } - free(pCtg); + taosMemoryFree(pCtg); } @@ -338,14 +338,14 @@ void ctgPopAction(SCtgMetaAction **action) { CTG_QUEUE_SUB(); - tfree(orig); + taosMemoryFreeClear(orig); *action = &node->action; } int32_t ctgPushAction(SCatalog* pCtg, SCtgMetaAction *action) { - SCtgQNode *node = calloc(1, sizeof(SCtgQNode)); + SCtgQNode *node = taosMemoryCalloc(1, sizeof(SCtgQNode)); if (NULL == node) { qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); CTG_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -378,7 +378,7 @@ int32_t ctgPushAction(SCatalog* pCtg, SCtgMetaAction *action) { int32_t ctgPushRmDBMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId) { int32_t code = 0; SCtgMetaAction action= {.act = CTG_ACT_REMOVE_DB}; - SCtgRemoveDBMsg *msg = malloc(sizeof(SCtgRemoveDBMsg)); + SCtgRemoveDBMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveDBMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveDBMsg)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -401,7 +401,7 @@ int32_t ctgPushRmDBMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId) _return: - tfree(action.data); + taosMemoryFreeClear(action.data); CTG_RET(code); } @@ -409,7 +409,7 @@ _return: int32_t ctgPushRmStbMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid) { int32_t code = 0; SCtgMetaAction action= {.act = CTG_ACT_REMOVE_STB}; - SCtgRemoveStbMsg *msg = malloc(sizeof(SCtgRemoveStbMsg)); + SCtgRemoveStbMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveStbMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveStbMsg)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -429,7 +429,7 @@ int32_t ctgPushRmStbMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId _return: - tfree(action.data); + taosMemoryFreeClear(action.data); CTG_RET(code); } @@ -438,7 +438,7 @@ _return: int32_t ctgPushRmTblMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *tbName) { int32_t code = 0; SCtgMetaAction action= {.act = CTG_ACT_REMOVE_TBL}; - SCtgRemoveTblMsg *msg = malloc(sizeof(SCtgRemoveTblMsg)); + SCtgRemoveTblMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveTblMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveTblMsg)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -457,14 +457,14 @@ int32_t ctgPushRmTblMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId _return: - tfree(action.data); + taosMemoryFreeClear(action.data); CTG_RET(code); } int32_t ctgPushUpdateVgMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, SDBVgInfo* dbInfo, bool syncReq) { int32_t code = 0; SCtgMetaAction action= {.act = CTG_ACT_UPDATE_VG, .syncReq = syncReq}; - SCtgUpdateVgMsg *msg = malloc(sizeof(SCtgUpdateVgMsg)); + SCtgUpdateVgMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateVgMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg)); ctgFreeVgInfo(dbInfo); @@ -490,14 +490,14 @@ int32_t ctgPushUpdateVgMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t d _return: ctgFreeVgInfo(dbInfo); - tfree(action.data); + taosMemoryFreeClear(action.data); CTG_RET(code); } int32_t ctgPushUpdateTblMsgInQueue(SCatalog* pCtg, STableMetaOutput *output, bool syncReq) { int32_t code = 0; SCtgMetaAction action= {.act = CTG_ACT_UPDATE_TBL}; - SCtgUpdateTblMsg *msg = malloc(sizeof(SCtgUpdateTblMsg)); + SCtgUpdateTblMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTblMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTblMsg)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -519,7 +519,7 @@ int32_t ctgPushUpdateTblMsgInQueue(SCatalog* pCtg, STableMetaOutput *output, boo _return: - tfree(msg); + taosMemoryFreeClear(msg); CTG_RET(code); } @@ -818,7 +818,7 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); ctgReleaseDBCache(pCtg, dbCache); ctgError("stb not in stbCache, suid:%"PRIx64, tbMeta->suid); - tfree(*pTableMeta); + taosMemoryFreeClear(*pTableMeta); *exist = 0; return TSDB_CODE_SUCCESS; } @@ -826,13 +826,13 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable if ((*stbMeta)->suid != tbMeta->suid) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); ctgReleaseDBCache(pCtg, dbCache); - tfree(*pTableMeta); + taosMemoryFreeClear(*pTableMeta); ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, tbMeta->suid, (*stbMeta)->suid); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } int32_t metaSize = CTG_META_SIZE(*stbMeta); - *pTableMeta = realloc(*pTableMeta, metaSize); + *pTableMeta = taosMemoryRealloc(*pTableMeta, metaSize); if (NULL == *pTableMeta) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); ctgReleaseDBCache(pCtg, dbCache); @@ -1161,7 +1161,7 @@ int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { size_t msgSize = sizeof(SCtgRentSlot) * mgmt->slotNum; - mgmt->slots = calloc(1, msgSize); + mgmt->slots = taosMemoryCalloc(1, msgSize); if (NULL == mgmt->slots) { qError("calloc %d failed", (int32_t)msgSize); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -1305,7 +1305,7 @@ int32_t ctgMetaRentGetImpl(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_ } size_t msize = metaNum * size; - *res = malloc(msize); + *res = taosMemoryMalloc(msize); if (NULL == *res) { qError("malloc %d failed", (int32_t)msize); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); @@ -1631,7 +1631,7 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui } int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) { - *dst = malloc(sizeof(SDBVgInfo)); + *dst = taosMemoryMalloc(sizeof(SDBVgInfo)); if (NULL == *dst) { qError("malloc %d failed", (int32_t)sizeof(SDBVgInfo)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -1643,7 +1643,7 @@ int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) { (*dst)->vgHash = taosHashInit(hashSize, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (NULL == (*dst)->vgHash) { qError("taosHashInit %d failed", (int32_t)hashSize); - tfree(*dst); + taosMemoryFreeClear(*dst); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -1656,7 +1656,7 @@ int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) { qError("taosHashPut failed, hashSize:%d", (int32_t)hashSize); taosHashCancelIterate(src->vgHash, pIter); taosHashCleanup((*dst)->vgHash); - tfree(*dst); + taosMemoryFreeClear(*dst); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -1703,7 +1703,7 @@ int32_t ctgGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const _return: - tfree(*pInfo); + taosMemoryFreeClear(*pInfo); *pInfo = DbOut.dbVgroup; CTG_RET(code); @@ -1748,7 +1748,7 @@ int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, c int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) { - *pOutput = malloc(sizeof(STableMetaOutput)); + *pOutput = taosMemoryMalloc(sizeof(STableMetaOutput)); if (NULL == *pOutput) { qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -1758,10 +1758,10 @@ int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) if (output->tbMeta) { int32_t metaSize = CTG_META_SIZE(output->tbMeta); - (*pOutput)->tbMeta = malloc(metaSize); + (*pOutput)->tbMeta = taosMemoryMalloc(metaSize); if (NULL == (*pOutput)->tbMeta) { qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); - tfree(*pOutput); + taosMemoryFreeClear(*pOutput); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -1786,7 +1786,7 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, } STableMetaOutput moutput = {0}; - STableMetaOutput *output = calloc(1, sizeof(STableMetaOutput)); + STableMetaOutput *output = taosMemoryCalloc(1, sizeof(STableMetaOutput)); if (NULL == output) { ctgError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -1814,7 +1814,7 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, if (CTG_IS_META_TABLE(output->metaType) && TSDB_SUPER_TABLE == output->tbMeta->tableType) { ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(pTableName)); - tfree(output->tbMeta); + taosMemoryFreeClear(output->tbMeta); CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, output->dbFName, output->tbName, output)); } else if (CTG_IS_META_BOTH(output->metaType)) { @@ -1830,11 +1830,11 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SET_META_TYPE_NULL(output->metaType); } - tfree(output->tbMeta); + taosMemoryFreeClear(output->tbMeta); output->tbMeta = moutput.tbMeta; moutput.tbMeta = NULL; } else { - tfree(output->tbMeta); + taosMemoryFreeClear(output->tbMeta); SET_META_TYPE_CTABLE(output->metaType); } @@ -1862,8 +1862,8 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, _return: - tfree(output->tbMeta); - tfree(output); + taosMemoryFreeClear(output->tbMeta); + taosMemoryFreeClear(output); CTG_RET(code); } @@ -1895,7 +1895,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons tbType = (*pTableMeta)->tableType; suid = (*pTableMeta)->suid; - tfree(*pTableMeta); + taosMemoryFreeClear(*pTableMeta); } if (CTG_FLAG_IS_UNKNOWN_STB(flag)) { @@ -1920,7 +1920,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons if ((!CTG_IS_META_CTABLE(output->metaType)) || output->tbMeta) { ctgError("invalid metaType:%d", output->metaType); - tfree(output->tbMeta); + taosMemoryFreeClear(output->tbMeta); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -1957,7 +1957,7 @@ _return: } } - tfree(output); + taosMemoryFreeClear(output); if (*pTableMeta) { ctgDebug("tbmeta returned, tbName:%s, tbType:%d", pTableName->tname, (*pTableMeta)->tableType); @@ -1978,7 +1978,7 @@ int32_t ctgActUpdateVg(SCtgMetaAction *action) { _return: ctgFreeVgInfo(msg->dbInfo); - tfree(msg); + taosMemoryFreeClear(msg); CTG_RET(code); } @@ -2003,7 +2003,7 @@ int32_t ctgActRemoveDB(SCtgMetaAction *action) { _return: - tfree(msg); + taosMemoryFreeClear(msg); CTG_RET(code); } @@ -2045,11 +2045,11 @@ int32_t ctgActUpdateTbl(SCtgMetaAction *action) { _return: if (output) { - tfree(output->tbMeta); - tfree(output); + taosMemoryFreeClear(output->tbMeta); + taosMemoryFreeClear(output); } - tfree(msg); + taosMemoryFreeClear(msg); CTG_RET(code); } @@ -2092,7 +2092,7 @@ int32_t ctgActRemoveStb(SCtgMetaAction *action) { _return: - tfree(msg); + taosMemoryFreeClear(msg); CTG_RET(code); } @@ -2125,7 +2125,7 @@ int32_t ctgActRemoveTbl(SCtgMetaAction *action) { _return: - tfree(msg); + taosMemoryFreeClear(msg); CTG_RET(code); } @@ -2248,11 +2248,11 @@ _return: ctgReleaseDBCache(pCtg, dbCache); } - tfree(tbMeta); + taosMemoryFreeClear(tbMeta); if (vgInfo) { taosHashCleanup(vgInfo->vgHash); - tfree(vgInfo); + taosMemoryFreeClear(vgInfo); } if (vgList) { @@ -2308,7 +2308,7 @@ int32_t catalogInit(SCatalogCfg *cfg) { tsem_init(&gCtgMgmt.queue.reqSem, 0, 0); tsem_init(&gCtgMgmt.queue.rspSem, 0, 0); - gCtgMgmt.queue.head = calloc(1, sizeof(SCtgQNode)); + gCtgMgmt.queue.head = taosMemoryCalloc(1, sizeof(SCtgQNode)); if (NULL == gCtgMgmt.queue.head) { qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); CTG_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -2342,7 +2342,7 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { return TSDB_CODE_SUCCESS; } - clusterCtg = calloc(1, sizeof(SCatalog)); + clusterCtg = taosMemoryCalloc(1, sizeof(SCatalog)); if (NULL == clusterCtg) { qError("calloc %d failed", (int32_t)sizeof(SCatalog)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -2483,7 +2483,7 @@ _return: if (vgInfo) { taosHashCleanup(vgInfo->vgHash); - tfree(vgInfo); + taosMemoryFreeClear(vgInfo); } CTG_API_LEAVE(code); @@ -2569,7 +2569,7 @@ int32_t catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName) { _return: - tfree(tblMeta); + taosMemoryFreeClear(tblMeta); CTG_API_LEAVE(code); } @@ -2620,7 +2620,7 @@ int32_t catalogUpdateSTableMeta(SCatalog* pCtg, STableMetaRsp *rspMsg) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - STableMetaOutput *output = calloc(1, sizeof(STableMetaOutput)); + STableMetaOutput *output = taosMemoryCalloc(1, sizeof(STableMetaOutput)); if (NULL == output) { ctgError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); CTG_API_LEAVE(TSDB_CODE_CTG_MEM_ERROR); @@ -2643,8 +2643,8 @@ int32_t catalogUpdateSTableMeta(SCatalog* pCtg, STableMetaRsp *rspMsg) { _return: - tfree(output->tbMeta); - tfree(output); + taosMemoryFreeClear(output->tbMeta); + taosMemoryFreeClear(output); CTG_API_LEAVE(code); } @@ -2739,7 +2739,7 @@ _return: if (vgInfo) { taosHashCleanup(vgInfo->vgHash); - tfree(vgInfo); + taosMemoryFreeClear(vgInfo); } CTG_API_LEAVE(code); @@ -2777,7 +2777,7 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) { ctgError("taosArrayPush failed, idx:%d", i); - tfree(pTableMeta); + taosMemoryFreeClear(pTableMeta); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } } @@ -2795,7 +2795,7 @@ _return: int32_t aSize = taosArrayGetSize(pRsp->pTableMeta); for (int32_t i = 0; i < aSize; ++i) { STableMeta *pMeta = taosArrayGetP(pRsp->pTableMeta, i); - tfree(pMeta); + taosMemoryFreeClear(pMeta); } taosArrayDestroy(pRsp->pTableMeta); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index aff2c2edcdad57e162da2f005efbcc7d36be6d37..e62819b07857aaab9bdedfbdaee6cfb2826aa6f8 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -176,7 +176,7 @@ void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) { output->ctbMeta.uid = 3; output->ctbMeta.suid = 2; - output->tbMeta = (STableMeta *)calloc(1, sizeof(STableMeta) + sizeof(SSchema) * (ctgTestColNum + ctgTestColNum)); + output->tbMeta = (STableMeta *)taosMemoryCalloc(1, sizeof(STableMeta) + sizeof(SSchema) * (ctgTestColNum + ctgTestColNum)); output->tbMeta->vgId = 9; output->tbMeta->tableType = TSDB_SUPER_TABLE; output->tbMeta->uid = 2; @@ -212,7 +212,7 @@ void ctgTestBuildDBVgroup(SDBVgInfo **pdbVgroup) { static int32_t vgVersion = ctgTestVgVersion + 1; int32_t vgNum = 0; SVgroupInfo vgInfo = {0}; - SDBVgInfo *dbVgroup = (SDBVgInfo *)calloc(1, sizeof(SDBVgInfo)); + SDBVgInfo *dbVgroup = (SDBVgInfo *)taosMemoryCalloc(1, sizeof(SDBVgInfo)); dbVgroup->vgVersion = vgVersion++; @@ -257,7 +257,7 @@ void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) { rspMsg->tuid = ctgTestSuid + 1; rspMsg->vgId = 1; - rspMsg->pSchemas = (SSchema *)calloc(rspMsg->numOfTags + rspMsg->numOfColumns, sizeof(SSchema)); + rspMsg->pSchemas = (SSchema *)taosMemoryCalloc(rspMsg->numOfTags + rspMsg->numOfColumns, sizeof(SSchema)); SSchema *s = NULL; s = &rspMsg->pSchemas[0]; @@ -335,7 +335,7 @@ void ctgTestRspTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg * metaRsp.suid = 0; metaRsp.tuid = ctgTestNormalTblUid++; metaRsp.vgId = 8; - metaRsp.pSchemas = (SSchema *)malloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); + metaRsp.pSchemas = (SSchema *)taosMemoryMalloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); SSchema *s = NULL; s = &metaRsp.pSchemas[0]; @@ -381,7 +381,7 @@ void ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg metaRsp.suid = 0x0000000000000002; metaRsp.tuid = 0x0000000000000003; metaRsp.vgId = 9; - metaRsp.pSchemas = (SSchema *)malloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); + metaRsp.pSchemas = (SSchema *)taosMemoryMalloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); SSchema *s = NULL; s = &metaRsp.pSchemas[0]; @@ -428,7 +428,7 @@ void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg metaRsp.suid = ctgTestSuid; metaRsp.tuid = ctgTestSuid++; metaRsp.vgId = 0; - metaRsp.pSchemas = (SSchema *)malloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); + metaRsp.pSchemas = (SSchema *)taosMemoryMalloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); SSchema *s = NULL; s = &metaRsp.pSchemas[0]; @@ -477,7 +477,7 @@ void ctgTestRspMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRp metaRsp.suid = ctgTestSuid + idx; metaRsp.tuid = ctgTestSuid + idx; metaRsp.vgId = 0; - metaRsp.pSchemas = (SSchema *)malloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); + metaRsp.pSchemas = (SSchema *)taosMemoryMalloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); SSchema *s = NULL; s = &metaRsp.pSchemas[0]; @@ -798,7 +798,7 @@ void *ctgTestGetCtableMetaThread(void *param) { assert(0); } - tfree(tbMeta); + taosMemoryFreeClear(tbMeta); if (ctgTestEnableSleep) { taosUsleep(taosRand() % 5); @@ -824,10 +824,10 @@ void *ctgTestSetCtableMetaThread(void *param) { action.act = CTG_ACT_UPDATE_TBL; while (!ctgTestStop) { - output = (STableMetaOutput *)malloc(sizeof(STableMetaOutput)); + output = (STableMetaOutput *)taosMemoryMalloc(sizeof(STableMetaOutput)); ctgTestBuildCTableMetaOutput(output); - SCtgUpdateTblMsg *msg = (SCtgUpdateTblMsg *)malloc(sizeof(SCtgUpdateTblMsg)); + SCtgUpdateTblMsg *msg = (SCtgUpdateTblMsg *)taosMemoryMalloc(sizeof(SCtgUpdateTblMsg)); msg->pCtg = pCtg; msg->output = output; action.data = msg; @@ -933,7 +933,7 @@ TEST(tableMeta, normalTable) { if (dbNum) { printf("got expired db,dbId:%" PRId64 "\n", dbs->dbId); - free(dbs); + taosMemoryFree(dbs); dbs = NULL; } else { printf("no expired db\n"); @@ -941,7 +941,7 @@ TEST(tableMeta, normalTable) { if (stbNum) { printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); - free(stb); + taosMemoryFree(stb); stb = NULL; } else { printf("no expired stb\n"); @@ -1015,7 +1015,7 @@ TEST(tableMeta, childTableCase) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); strcpy(n.tname, ctgTestSTablename); code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); @@ -1042,7 +1042,7 @@ TEST(tableMeta, childTableCase) { if (dbNum) { printf("got expired db,dbId:%" PRId64 "\n", dbs->dbId); - free(dbs); + taosMemoryFree(dbs); dbs = NULL; } else { printf("no expired db\n"); @@ -1050,7 +1050,7 @@ TEST(tableMeta, childTableCase) { if (stbNum) { printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); - free(stb); + taosMemoryFree(stb); stb = NULL; } else { printf("no expired stb\n"); @@ -1164,7 +1164,7 @@ TEST(tableMeta, superTableCase) { if (dbNum) { printf("got expired db,dbId:%" PRId64 "\n", dbs->dbId); - free(dbs); + taosMemoryFree(dbs); dbs = NULL; } else { printf("no expired db\n"); @@ -1173,7 +1173,7 @@ TEST(tableMeta, superTableCase) { if (stbNum) { printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); - free(stb); + taosMemoryFree(stb); stb = NULL; } else { printf("no expired stb\n"); @@ -1307,14 +1307,14 @@ TEST(tableMeta, updateStbMeta) { } - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); STableMetaRsp rsp = {0}; ctgTestBuildSTableMetaRsp(&rsp); code = catalogUpdateSTableMeta(pCtg, &rsp); ASSERT_EQ(code, 0); - tfree(rsp.pSchemas); + taosMemoryFreeClear(rsp.pSchemas); while (true) { uint64_t n = 0; @@ -1345,7 +1345,7 @@ TEST(tableMeta, updateStbMeta) { ASSERT_EQ(tableMeta->tableInfo.precision, 1 + 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); catalogDestroy(); memset(&gCtgMgmt.stat, 0, sizeof(gCtgMgmt.stat)); @@ -1407,7 +1407,7 @@ TEST(refreshGetMeta, normal2normal) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, 0); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); @@ -1424,7 +1424,7 @@ TEST(refreshGetMeta, normal2normal) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, 0); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -1486,7 +1486,7 @@ TEST(refreshGetMeta, normal2notexist) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, 0); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); @@ -1560,7 +1560,7 @@ TEST(refreshGetMeta, normal2child) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, 0); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); @@ -1576,7 +1576,7 @@ TEST(refreshGetMeta, normal2child) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -1645,7 +1645,7 @@ TEST(refreshGetMeta, stable2child) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); @@ -1662,7 +1662,7 @@ TEST(refreshGetMeta, stable2child) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -1730,7 +1730,7 @@ TEST(refreshGetMeta, stable2stable) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); @@ -1748,7 +1748,7 @@ TEST(refreshGetMeta, stable2stable) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -1816,7 +1816,7 @@ TEST(refreshGetMeta, child2stable) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); while (2 != ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); @@ -1835,7 +1835,7 @@ TEST(refreshGetMeta, child2stable) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -2275,7 +2275,7 @@ TEST(rentTest, allRent) { printf("%d - expired dbNum:%d\n", i, num); if (dbs) { printf("%d - expired dbId:%" PRId64 ", vgVersion:%d\n", i, dbs->dbId, dbs->vgVersion); - free(dbs); + taosMemoryFree(dbs); dbs = NULL; } @@ -2287,7 +2287,7 @@ TEST(rentTest, allRent) { printf("suid:%" PRId64 ", dbFName:%s, stbName:%s, sversion:%d, tversion:%d\n", stable[n].suid, stable[n].dbFName, stable[n].stbName, stable[n].sversion, stable[n].tversion); } - free(stable); + taosMemoryFree(stable); stable = NULL; } printf("*************************************************\n"); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index f170d5c52904badc0e63bc98fa2086ac25972839..c582873315d38efb14e904e436bd0ef3b055270f 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -76,11 +76,12 @@ typedef struct SResultRowCell { * If the number of generated results is greater than this value, * query query will be halt and return results to client immediate. */ -typedef struct SRspResultInfo { - int64_t total; // total generated result size in rows - int32_t capacity; // capacity of current result output buffer - int32_t threshold; // result size threshold in rows. -} SRspResultInfo; +typedef struct SResultInfo { // TODO refactor + int64_t totalRows; // total generated result size in rows + int64_t totalBytes; // total results in bytes. + int32_t capacity; // capacity of current result output buffer + int32_t threshold; // result size threshold in rows. +} SResultInfo; typedef struct SColumnFilterElem { int16_t bytes; // column length @@ -160,8 +161,8 @@ typedef struct STaskCostInfo { typedef struct SOperatorCostInfo { uint64_t openCost; uint64_t execCost; - uint64_t totalRows; - uint64_t totalBytes; +// uint64_t totalRows; +// uint64_t totalBytes; } SOperatorCostInfo; typedef struct { @@ -242,6 +243,8 @@ typedef struct STaskAttr { struct SOperatorInfo; +typedef void (*__optr_encode_fn_t)(struct SOperatorInfo* pOperator, char **result, int32_t *length); +typedef bool (*__optr_decode_fn_t)(struct SOperatorInfo* pOperator, char *result, int32_t length); typedef int32_t (*__optr_open_fn_t)(struct SOperatorInfo* param); typedef SSDataBlock* (*__optr_fn_t)(struct SOperatorInfo* param, bool* newgroup); typedef void (*__optr_close_fn_t)(void* param, int32_t num); @@ -301,7 +304,7 @@ typedef struct STaskRuntimeEnv { int64_t currentOffset; // dynamic offset value STableQueryInfo* current; - SRspResultInfo resultInfo; + SResultInfo resultInfo; SHashObj* pTableRetrieveTsMap; struct SUdfInfo* pUdfInfo; } STaskRuntimeEnv; @@ -324,13 +327,15 @@ typedef struct SOperatorInfo { STaskRuntimeEnv* pRuntimeEnv; // todo remove it SExecTaskInfo* pTaskInfo; SOperatorCostInfo cost; - + SResultInfo resultInfo; struct SOperatorInfo** pDownstream; // downstram pointer list int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator __optr_fn_t getNextFn; __optr_fn_t cleanupFn; __optr_close_fn_t closeFn; __optr_open_fn_t _openFn; // DO NOT invoke this function directly + __optr_encode_fn_t encodeResultRow; // + __optr_decode_fn_t decodeResultRow; } SOperatorInfo; typedef struct { @@ -539,12 +544,24 @@ typedef struct SFillOperatorInfo { void** p; SSDataBlock* existNewGroupBlock; bool multigroupResult; + SInterval intervalInfo; + int32_t capacity; } SFillOperatorInfo; +typedef struct SGroupKeys { + char *pData; + bool isNull; + int16_t type; + int32_t bytes; +}SGroupKeys; + typedef struct SGroupbyOperatorInfo { SOptrBasicInfo binfo; SArray* pGroupCols; - char* prevData; // previous group by value + SArray* pGroupColVals; // current group column values, SArray + bool isInit; // denote if current val is initialized or not + char* keyBuf; // group by keys for hash + int32_t groupKeyLen; // total group by column width SGroupResInfo groupResInfo; SAggSupporter aggSup; } SGroupbyOperatorInfo; @@ -639,29 +656,29 @@ SOperatorInfo* createOrderOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pOrderVal, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataBlock* pResBlock, const SName* pName, SNode* pCondition, SEpSet epset, SArray* colList, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createLimitOperatorInfo(SOperatorInfo* downstream, int32_t numOfDownstream, SLimit* pLimit, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createLimitOperatorInfo(SOperatorInfo* downstream, SLimit* pLimit, SExecTaskInfo* pTaskInfo); SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo); SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SArray* pGroupColList, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); +SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SInterval* pInterval, SSDataBlock* pResBlock, + int32_t fillType, char* fillVal, bool multigroupResult, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, + int32_t numOfOutput); +SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, - int32_t numOfOutput, bool multigroupResult); - SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, - int32_t numOfOutput); -SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); + SOperatorInfo* createMultiwaySortOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows, void* merger); SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 1c2d9424d3e174481a6a8e0a7a41f6939c30e952..34894c235b0e0565710a2b8d1908ff5a6b6e3fee 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -142,7 +142,7 @@ static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, pBuf->allocSize = sizeof(SRetrieveTableRsp) + blockDataGetSerialMetaSize(pInput->pData) + ceil(blockDataGetSerialRowSize(pInput->pData) * pInput->pData->info.rows); - pBuf->pData = malloc(pBuf->allocSize); + pBuf->pData = taosMemoryMalloc(pBuf->allocSize); if (pBuf->pData == NULL) { qError("SinkNode failed to malloc memory, size:%d, code:%d", pBuf->allocSize, TAOS_SYSTEM_ERROR(errno)); } @@ -215,7 +215,7 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { memcpy(pOutput->pData, pEntry->data, pEntry->dataLen); pOutput->numOfRows = pEntry->numOfRows; pOutput->compressed = pEntry->compressed; - tfree(pDispatcher->nextOutput.pData); // todo persistent + taosMemoryFreeClear(pDispatcher->nextOutput.pData); // todo persistent pOutput->bufStatus = updateStatus(pDispatcher); taosThreadMutexLock(&pDispatcher->mutex); pOutput->queryEnd = pDispatcher->queryEnd; @@ -227,11 +227,11 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; - tfree(pDispatcher->nextOutput.pData); + taosMemoryFreeClear(pDispatcher->nextOutput.pData); while (!taosQueueEmpty(pDispatcher->pDataBlocks)) { SDataDispatchBuf* pBuf = NULL; taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); - tfree(pBuf->pData); + taosMemoryFreeClear(pBuf->pData); taosFreeQitem(pBuf); } taosCloseQueue(pDispatcher->pDataBlocks); @@ -239,7 +239,7 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { } int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle) { - SDataDispatchHandle* dispatcher = calloc(1, sizeof(SDataDispatchHandle)); + SDataDispatchHandle* dispatcher = taosMemoryCalloc(1, sizeof(SDataDispatchHandle)); if (NULL == dispatcher) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return TSDB_CODE_QRY_OUT_OF_MEMORY; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index a04a10ef95242ef59b74001a1ea07d7cc004fd24..f0cffafca2fcc053969707a181832a5d005b7124 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -58,8 +58,8 @@ int32_t initResultRowInfo(SResultRowInfo *pResultRowInfo, int32_t size) { pResultRowInfo->curPos = -1; pResultRowInfo->capacity = size; - pResultRowInfo->pResult = calloc(pResultRowInfo->capacity, POINTER_BYTES); - pResultRowInfo->pPosition = calloc(pResultRowInfo->capacity, sizeof(SResultRowPosition)); + pResultRowInfo->pResult = taosMemoryCalloc(pResultRowInfo->capacity, POINTER_BYTES); + pResultRowInfo->pPosition = taosMemoryCalloc(pResultRowInfo->capacity, sizeof(SResultRowPosition)); if (pResultRowInfo->pResult == NULL || pResultRowInfo->pPosition == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -79,11 +79,11 @@ void cleanupResultRowInfo(SResultRowInfo *pResultRowInfo) { for(int32_t i = 0; i < pResultRowInfo->size; ++i) { if (pResultRowInfo->pResult[i]) { - tfree(pResultRowInfo->pResult[i]->key); + taosMemoryFreeClear(pResultRowInfo->pResult[i]->key); } } - tfree(pResultRowInfo->pResult); + taosMemoryFreeClear(pResultRowInfo->pResult); } void resetResultRowInfo(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo) { @@ -163,7 +163,7 @@ void clearResultRow(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow) { pResultRow->offset = -1; pResultRow->closed = false; - tfree(pResultRow->key); + taosMemoryFreeClear(pResultRow->key); pResultRow->win = TSWINDOW_INITIALIZER; } @@ -190,7 +190,7 @@ SResultRow* getNewResultRow(SResultRowPool* p) { void* ptr = NULL; if (p->position.pos == 0) { - ptr = calloc(1, p->blockSize); + ptr = taosMemoryCalloc(1, p->blockSize); taosArrayPush(p->pData, &ptr); } else { @@ -403,8 +403,8 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv pGroupResInfo->pRows = taosArrayInit(100, POINTER_BYTES); } - posList = calloc(size, sizeof(int32_t)); - pTableQueryInfoList = malloc(POINTER_BYTES * size); + posList = taosMemoryCalloc(size, sizeof(int32_t)); + pTableQueryInfoList = taosMemoryMalloc(POINTER_BYTES * size); if (pTableQueryInfoList == NULL || posList == NULL || pGroupResInfo->pRows == NULL || pGroupResInfo->pRows == NULL) { // qError("QInfo:%"PRIu64" failed alloc memory", GET_TASKID(pRuntimeEnv)); @@ -483,9 +483,9 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv // pGroupResInfo->currentGroup, endt - startt); _end: - tfree(pTableQueryInfoList); - tfree(posList); - tfree(pTree); + taosMemoryFreeClear(pTableQueryInfoList); + taosMemoryFreeClear(posList); + taosMemoryFreeClear(pTree); return code; } @@ -529,7 +529,7 @@ int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, STaskRuntimeEnv* pRun // // // compress extra bytes // size_t x = taosArrayGetSize(pDist->dataBlockInfos) * pDist->dataBlockInfos->elemSize; -// char* tmp = malloc(x + 2); +// char* tmp = taosMemoryMalloc(x + 2); // // bool comp = false; // int32_t len = tsCompressString(p, (int32_t)x, 1, tmp, (int32_t)x, ONE_STAGE_COMP, NULL, 0); @@ -547,7 +547,7 @@ int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, STaskRuntimeEnv* pRun // } else { // tbufWriteBinary(bw, p, len); // } -// tfree(tmp); +// taosMemoryFreeClear(tmp); //} //void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDist) { @@ -570,7 +570,7 @@ int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, STaskRuntimeEnv* pRun // // char* outputBuf = NULL; // if (comp) { -// outputBuf = malloc(originalLen); +// outputBuf = taosMemoryMalloc(originalLen); // // size_t actualLen = compLen; // const char* compStr = tbufReadBinary(&br, &actualLen); @@ -584,7 +584,7 @@ int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, STaskRuntimeEnv* pRun // // pDist->dataBlockInfos = taosArrayFromList(outputBuf, (uint32_t)numSteps, sizeof(SFileBlockInfo)); // if (comp) { -// tfree(outputBuf); +// taosMemoryFreeClear(outputBuf); // } //} diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 19100d7560b6eb3df1519f479b827294891865b9..e89bc5df0e52940e19bfe69c3d17b46bd79edb02 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -30,7 +30,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, int32_t t qError("join not supported for stream block scan, %s" PRIx64, id); return TSDB_CODE_QRY_APP_ERROR; } - + pOperator->status = OP_NOT_OPENED; return doSetStreamBlock(pOperator->pDownstream[0], input, type, id); } else { SStreamBlockScanInfo* pInfo = pOperator->info; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index e927b577deb3d37cedd8e7e101e92bda5ba6ea6f..fc0535358b6e24e861e7b344771340805a7b8eeb 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -73,7 +73,7 @@ static UNUSED_FUNC void *u_malloc (size_t __size) { if (v % 1000 <= 0) { return NULL; } else { - return malloc(__size); + return taosMemoryMalloc(__size); } } @@ -82,7 +82,7 @@ static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) { if (v % 1000 <= 0) { return NULL; } else { - return calloc(num, __size); + return taosMemoryCalloc(num, __size); } } @@ -91,7 +91,7 @@ static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { if (v % 5 <= 1) { return NULL; } else { - return realloc(p, __size); + return taosMemoryRealloc(p, __size); } } @@ -248,7 +248,7 @@ static int32_t setGroupResultOutputBuf_rv(SOptrBasicInfo *binfo, int32_t numOfCo static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); static void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win); -static void setResultBufSize(STaskAttr* pQueryAttr, SRspResultInfo* pResultInfo); +static void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo); static void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); static void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr); static void setParamForStableStddevByColData(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes); @@ -320,7 +320,7 @@ static void sortGroupResByOrderList(SGroupResInfo *pGroupResInfo, STaskRuntimeEn SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows) { const static int32_t minSize = 8; - SSDataBlock *res = calloc(1, sizeof(SSDataBlock)); + SSDataBlock *res = taosMemoryCalloc(1, sizeof(SSDataBlock)); res->info.numOfCols = numOfOutput; res->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); for (int32_t i = 0; i < numOfOutput; ++i) { @@ -330,7 +330,7 @@ SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numO idata.info.colId = pExpr[i].base.resSchema.colId; int32_t size = TMAX(idata.info.bytes * numOfRows, minSize); - idata.pData = calloc(1, size); // at least to hold a pointer on x64 platform + idata.pData = taosMemoryCalloc(1, size); // at least to hold a pointer on x64 platform taosArrayPush(res->pDataBlock, &idata); } @@ -340,7 +340,7 @@ SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numO SSDataBlock* createOutputBuf_rv1(SDataBlockDescNode* pNode) { int32_t numOfCols = LIST_LENGTH(pNode->pSlots); - SSDataBlock* pBlock = calloc(1, sizeof(SSDataBlock)); + SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); pBlock->info.numOfCols = numOfCols; pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); @@ -423,12 +423,12 @@ static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, jmp_buf env) newCapacity += 4; } - char *t = realloc(pResultRowInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); + char *t = taosMemoryRealloc(pResultRowInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); if (t == NULL) { longjmp(env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - pResultRowInfo->pPosition = realloc(pResultRowInfo->pPosition, newCapacity * sizeof(SResultRowPosition)); + pResultRowInfo->pPosition = taosMemoryRealloc(pResultRowInfo->pPosition, newCapacity * sizeof(SResultRowPosition)); pResultRowInfo->pResult = (SResultRow **)t; int32_t inc = (int32_t)newCapacity - pResultRowInfo->capacity; @@ -674,12 +674,7 @@ static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, T int64_t key = w->skey; while(key < ts) { // moving towards end - if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - key = taosTimeAdd(key, pInterval->sliding, pInterval->slidingUnit, precision); - } else { - key += pInterval->sliding; - } - + key = taosTimeAdd(key, pInterval->sliding, pInterval->slidingUnit, precision); if (key >= ts) { break; } @@ -695,12 +690,7 @@ static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t if (pResultRowInfo->curPos == -1) { // the first window, from the previous stored value getInitialStartTimeWindow(pInterval, precision, ts, &w, win->ekey, true); - - if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; - } else { - w.ekey = w.skey + pInterval->interval - 1; - } + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { w = getResultRow(pResultRowInfo, pResultRowInfo->curPos)->win; } @@ -722,7 +712,7 @@ static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t } w.skey = st; - w.ekey = w.skey + pInterval->interval - 1; + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } } return w; @@ -1623,76 +1613,176 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe // updateResultRowInfoActiveIndex(pResultRowInfo, pQueryAttr, pRuntimeEnv->current->lastKey); } -static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pInfo, SSDataBlock *pBlock) { - SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; +static bool groupKeyCompare(SGroupbyOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t rowIndex, int32_t numOfGroupCols) { + SColumnDataAgg* pColAgg = NULL; + for (int32_t i = 0; i < numOfGroupCols; ++i) { + SColumn* pCol = taosArrayGet(pInfo->pGroupCols, i); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); + if (pBlock->pBlockAgg != NULL) { + pColAgg = &pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? + } - // TODO multiple group by columns - SColumn* pCol = taosArrayGet(pInfo->pGroupCols, 0); - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); + bool isNull = colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg); - int16_t bytes = pColInfoData->info.bytes; - int16_t type = pColInfoData->info.type; - if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { - //qError("QInfo:0x%"PRIx64" group by not supported on double/float columns, abort", GET_TASKID(pRuntimeEnv)); - return; + SGroupKeys* pkey = taosArrayGet(pInfo->pGroupColVals, i); + if (pkey->isNull && isNull) { + continue; + } + + if (isNull || pkey->isNull) { + return false; + } + + char* val = colDataGetData(pColInfoData, rowIndex); + + if (IS_VAR_DATA_TYPE(pkey->type)) { + int32_t len = varDataLen(val); + if (len == varDataLen(pkey->pData) && memcmp(varDataVal(pkey->pData), varDataVal(val), len) == 0) { + continue; + } else { + return false; + } + } else { + if (memcmp(pkey->pData, val, pkey->bytes) != 0) { + return false; + } + } } - STimeWindow w = TSWINDOW_INITIALIZER; + return true; +} - int32_t num = 0; - for (int32_t j = 0; j < pBlock->info.rows; ++j) { - if (colDataIsNull(pColInfoData, pBlock->info.rows, j, NULL)) { // TODO +static void keepGroupKeys(SGroupbyOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t rowIndex, int32_t numOfGroupCols) { + SColumnDataAgg* pColAgg = NULL; + + for (int32_t i = 0; i < numOfGroupCols; ++i) { + SColumn* pCol = taosArrayGet(pInfo->pGroupCols, i); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); + + if (pBlock->pBlockAgg != NULL) { + pColAgg = &pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? + } + + SGroupKeys* pkey = taosArrayGet(pInfo->pGroupColVals, i); + if (colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg)) { + pkey->isNull = true; + } else { + char* val = colDataGetData(pColInfoData, rowIndex); + if (IS_VAR_DATA_TYPE(pkey->type)) { + memcpy(pkey->pData, val, varDataTLen(val)); + } else { + memcpy(pkey->pData, val, pkey->bytes); + } + } + } +} + +static int32_t generatedHashKey(void* pKey, int32_t* length, SArray* pGroupColVals) { + ASSERT(pKey != NULL); + size_t numOfGroupCols = taosArrayGetSize(pGroupColVals); + + char* isNull = (char*) pKey; + char* pStart = (char*) pKey + sizeof(int8_t) * numOfGroupCols; + for(int32_t i = 0; i < numOfGroupCols; ++i) { + SGroupKeys* pkey = taosArrayGet(pGroupColVals, i); + if (pkey->isNull) { + isNull[i] = 1; continue; } - char* val = colDataGetData(pColInfoData, j); + isNull[i] = 0; + if (IS_VAR_DATA_TYPE(pkey->type)) { + varDataCopy(pStart, pkey->pData); + pStart += varDataTLen(pkey->pData); + ASSERT(varDataTLen(pkey->pData) <= pkey->bytes); + } else { + memcpy(pStart, pkey->pData, pkey->bytes); + pStart += pkey->bytes; + } + } + + *length = (pStart - (char*) pKey); + return 0; +} + +// assign the group keys or user input constant values if required +static void doAssignGroupKeys(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t totalRows, int32_t rowIndex) { + for(int32_t i = 0; i < numOfOutput; ++i) { + if (pCtx[i].functionId == -1) { + SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pCtx[i]); + + SColumnInfoData* pColInfoData = pCtx[i].input.pData[0]; + if (!colDataIsNull(pColInfoData, totalRows, rowIndex, NULL)) { + char* dest = GET_ROWCELL_INTERBUF(pEntryInfo); + char* data = colDataGetData(pColInfoData, rowIndex); + + // set result exists, todo refactor + memcpy(dest, data, pColInfoData->info.bytes); + pEntryInfo->hasResult = DATA_SET_FLAG; + pEntryInfo->numOfRes = 1; + } + } + } +} + +static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock *pBlock) { + SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; + SGroupbyOperatorInfo *pInfo = pOperator->info; + + SqlFunctionCtx* pCtx = pInfo->binfo.pCtx; + int32_t numOfGroupCols = taosArrayGetSize(pInfo->pGroupCols); +// if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { + //qError("QInfo:0x%"PRIx64" group by not supported on double/float columns, abort", GET_TASKID(pRuntimeEnv)); +// return; +// } + int32_t len = 0; + STimeWindow w = TSWINDOW_INITIALIZER; + + int32_t num = 0; + for (int32_t j = 0; j < pBlock->info.rows; ++j) { // Compare with the previous row of this column, and do not set the output buffer again if they are identical. - if (pInfo->prevData == NULL) { - pInfo->prevData = malloc(bytes); - memcpy(pInfo->prevData, val, bytes); + if (!pInfo->isInit) { + keepGroupKeys(pInfo, pBlock, j, numOfGroupCols); + pInfo->isInit = true; num++; continue; } - if (IS_VAR_DATA_TYPE(type)) { - int32_t len = varDataLen(val); - if(len == varDataLen(pInfo->prevData) && memcmp(varDataVal(pInfo->prevData), varDataVal(val), len) == 0) { - num++; - continue; - } - } else { - if (memcmp(pInfo->prevData, val, bytes) == 0) { - num++; - continue; - } + bool equal = groupKeyCompare(pInfo, pBlock, j, numOfGroupCols); + if (equal) { + num++; + continue; } - int32_t ret = setGroupResultOutputBuf_rv(&(pInfo->binfo), pOperator->numOfOutput, pInfo->prevData, type, bytes, 0, - pInfo->aggSup.pResultBuf, pTaskInfo, &pInfo->aggSup); + /*int32_t ret = */generatedHashKey(pInfo->keyBuf, &len, pInfo->pGroupColVals); + int32_t ret = setGroupResultOutputBuf_rv(&(pInfo->binfo), pOperator->numOfOutput, pInfo->keyBuf, TSDB_DATA_TYPE_VARCHAR, len, 0, + pInfo->aggSup.pResultBuf, pTaskInfo, &pInfo->aggSup); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } - doApplyFunctions(pInfo->binfo.pCtx, &w, j - num, num, NULL, pBlock->info.rows, pOperator->numOfOutput, TSDB_ORDER_ASC); + int32_t rowIndex = j - num; + doApplyFunctions(pCtx, &w, rowIndex, num, NULL, pBlock->info.rows, pOperator->numOfOutput, TSDB_ORDER_ASC); + // assign the group keys or user input constant values if required + doAssignGroupKeys(pCtx, pOperator->numOfOutput, pBlock->info.rows, rowIndex); + keepGroupKeys(pInfo, pBlock, j, numOfGroupCols); num = 1; - memcpy(pInfo->prevData, val, bytes); } if (num > 0) { - char* val = ((char*)pColInfoData->pData) + bytes * (pBlock->info.rows - num); - memcpy(pInfo->prevData, val, bytes); - int32_t ret = setGroupResultOutputBuf_rv(&(pInfo->binfo), pOperator->numOfOutput, pInfo->prevData, type, bytes, 0, + /*int32_t ret = */generatedHashKey(pInfo->keyBuf, &len, pInfo->pGroupColVals); + int32_t ret = setGroupResultOutputBuf_rv(&(pInfo->binfo), pOperator->numOfOutput, pInfo->keyBuf, TSDB_DATA_TYPE_VARCHAR, len, 0, pInfo->aggSup.pResultBuf, pTaskInfo, &pInfo->aggSup); if (ret != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } - doApplyFunctions(pInfo->binfo.pCtx, &w, pBlock->info.rows - num, num, NULL, pBlock->info.rows, pOperator->numOfOutput, TSDB_ORDER_ASC); + int32_t rowIndex = pBlock->info.rows - num; + doApplyFunctions(pCtx, &w, rowIndex, num, NULL, pBlock->info.rows, pOperator->numOfOutput, TSDB_ORDER_ASC); + doAssignGroupKeys(pCtx, pOperator->numOfOutput, pBlock->info.rows, rowIndex); } - - tfree(pInfo->prevData); } static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { @@ -1767,7 +1857,7 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { if (IS_VAR_DATA_TYPE(type)) { if (pResultRow->key == NULL) { - pResultRow->key = malloc(varDataTLen(pData)); + pResultRow->key = taosMemoryMalloc(varDataTLen(pData)); varDataCopy(pResultRow->key, pData); } else { assert(memcmp(pResultRow->key, pData, varDataTLen(pData)) == 0); @@ -1783,20 +1873,11 @@ static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { static int32_t setGroupResultOutputBuf_rv(SOptrBasicInfo *binfo, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupId, SDiskbasedBuf* pBuf, SExecTaskInfo* pTaskInfo, SAggSupporter* pAggSup) { - int32_t *rowCellInfoOffset = binfo->rowCellInfoOffset; SResultRowInfo *pResultRowInfo = &binfo->resultRowInfo; SqlFunctionCtx *pCtx = binfo->pCtx; - // not assign result buffer yet, add new result buffer, TODO remove it - char* d = pData; - int16_t len = bytes; - if (IS_VAR_DATA_TYPE(type)) { - d = varDataVal(pData); - len = varDataLen(pData); - } - - int64_t tid = 0; - SResultRow *pResultRow = doSetResultOutBufByKey_rv(pBuf, pResultRowInfo, groupId, (char *)pData, TSDB_KEYSIZE, true, groupId, pTaskInfo, true, pAggSup); + SResultRow *pResultRow = doSetResultOutBufByKey_rv(pBuf, pResultRowInfo, groupId, (char *)pData, bytes, true, groupId, + pTaskInfo, true, pAggSup); assert (pResultRow != NULL); setResultRowKey(pResultRow, pData, type); @@ -1895,7 +1976,7 @@ static int32_t setCtxTagColumnInfo(SqlFunctionCtx *pCtx, int32_t numOfOutput) { int16_t tagLen = 0; SqlFunctionCtx* p = NULL; - SqlFunctionCtx** pTagCtx = calloc(numOfOutput, POINTER_BYTES); + SqlFunctionCtx** pTagCtx = taosMemoryCalloc(numOfOutput, POINTER_BYTES); if (pTagCtx == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -1921,7 +2002,7 @@ static int32_t setCtxTagColumnInfo(SqlFunctionCtx *pCtx, int32_t numOfOutput) { p->subsidiaryRes.numOfCols = num; p->subsidiaryRes.bufLen = tagLen; } else { - tfree(pTagCtx); + taosMemoryFreeClear(pTagCtx); } return TSDB_CODE_SUCCESS; @@ -1931,14 +2012,14 @@ static SqlFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprI int32_t** rowCellInfoOffset) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - SqlFunctionCtx * pFuncCtx = (SqlFunctionCtx *)calloc(numOfOutput, sizeof(SqlFunctionCtx)); + SqlFunctionCtx * pFuncCtx = (SqlFunctionCtx *)taosMemoryCalloc(numOfOutput, sizeof(SqlFunctionCtx)); if (pFuncCtx == NULL) { return NULL; } - *rowCellInfoOffset = calloc(numOfOutput, sizeof(int32_t)); + *rowCellInfoOffset = taosMemoryCalloc(numOfOutput, sizeof(int32_t)); if (*rowCellInfoOffset == 0) { - tfree(pFuncCtx); + taosMemoryFreeClear(pFuncCtx); return NULL; } @@ -2029,14 +2110,14 @@ static SqlFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprI } static SqlFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowCellInfoOffset) { - SqlFunctionCtx * pFuncCtx = (SqlFunctionCtx *)calloc(numOfOutput, sizeof(SqlFunctionCtx)); + SqlFunctionCtx * pFuncCtx = (SqlFunctionCtx *)taosMemoryCalloc(numOfOutput, sizeof(SqlFunctionCtx)); if (pFuncCtx == NULL) { return NULL; } - *rowCellInfoOffset = calloc(numOfOutput, sizeof(int32_t)); + *rowCellInfoOffset = taosMemoryCalloc(numOfOutput, sizeof(int32_t)); if (*rowCellInfoOffset == 0) { - tfree(pFuncCtx); + taosMemoryFreeClear(pFuncCtx); return NULL; } @@ -2058,8 +2139,8 @@ static SqlFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExprInfo, int32_t num } pCtx->input.numOfInputCols = pFunct->numOfParams; - pCtx->input.pData = calloc(pFunct->numOfParams, POINTER_BYTES); - pCtx->input.pColumnDataAgg = calloc(pFunct->numOfParams, POINTER_BYTES); + pCtx->input.pData = taosMemoryCalloc(pFunct->numOfParams, POINTER_BYTES); + pCtx->input.pColumnDataAgg = taosMemoryCalloc(pFunct->numOfParams, POINTER_BYTES); pCtx->ptsOutputBuf = NULL; pCtx->resDataInfo.bytes = pFunct->resSchema.bytes; @@ -2117,7 +2198,7 @@ static SqlFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExprInfo, int32_t num } for(int32_t i = 1; i < numOfOutput; ++i) { - (*rowCellInfoOffset)[i] = (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + pFuncCtx[i].resDataInfo.interBufSize); + (*rowCellInfoOffset)[i] = (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + pFuncCtx[i - 1].resDataInfo.interBufSize); } setCtxTagColumnInfo(pFuncCtx, numOfOutput); @@ -2135,10 +2216,10 @@ static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } taosVariantDestroy(&pCtx[i].tag); - tfree(pCtx[i].subsidiaryRes.pCtx); + taosMemoryFreeClear(pCtx[i].subsidiaryRes.pCtx); } - tfree(pCtx); + taosMemoryFreeClear(pCtx); return NULL; } @@ -2151,12 +2232,12 @@ static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfT pRuntimeEnv->pResultRowHashTable = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); pRuntimeEnv->pResultRowListSet = taosHashInit(numOfTables * 10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - pRuntimeEnv->keyBuf = malloc(pQueryAttr->maxTableColumnWidth + sizeof(int64_t) + POINTER_BYTES); + pRuntimeEnv->keyBuf = taosMemoryMalloc(pQueryAttr->maxTableColumnWidth + sizeof(int64_t) + POINTER_BYTES); // pRuntimeEnv->pool = initResultRowPool(getResultRowSize(pRuntimeEnv)); pRuntimeEnv->pResultRowArrayList = taosArrayInit(numOfTables, sizeof(SResultRowCell)); - pRuntimeEnv->prevRow = malloc(POINTER_BYTES * pQueryAttr->numOfCols + pQueryAttr->srcRowSize); - pRuntimeEnv->tagVal = malloc(pQueryAttr->tagLen); + pRuntimeEnv->prevRow = taosMemoryMalloc(POINTER_BYTES * pQueryAttr->numOfCols + pQueryAttr->srcRowSize); + pRuntimeEnv->tagVal = taosMemoryMalloc(pQueryAttr->tagLen); // NOTE: pTableCheckInfo need to update the query time range and the lastKey info pRuntimeEnv->pTableRetrieveTsMap = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); @@ -2188,10 +2269,10 @@ static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfT _clean: //destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pRuntimeEnv->pQueryAttr->numOfOutput); - tfree(pRuntimeEnv->pResultRowHashTable); - tfree(pRuntimeEnv->keyBuf); - tfree(pRuntimeEnv->prevRow); - tfree(pRuntimeEnv->tagVal); + taosMemoryFreeClear(pRuntimeEnv->pResultRowHashTable); + taosMemoryFreeClear(pRuntimeEnv->keyBuf); + taosMemoryFreeClear(pRuntimeEnv->prevRow); + taosMemoryFreeClear(pRuntimeEnv->tagVal); return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -2316,7 +2397,7 @@ static bool isCachedLastQuery(STaskAttr *pQueryAttr) { ///////////////////////////////////////////////////////////////////////////////////////////// //todo refactor : return window void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win) { - assert(key >= keyFirst && key <= keyLast && pInterval->sliding <= pInterval->interval); + ASSERT(key >= keyFirst && key <= keyLast); win->skey = taosTimeTruncate(key, pInterval, precision); /* @@ -2326,10 +2407,8 @@ void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t ke if (keyFirst > (INT64_MAX - pInterval->interval)) { assert(keyLast - keyFirst < pInterval->interval); win->ekey = INT64_MAX; - } else if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - win->ekey = taosTimeAdd(win->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { - win->ekey = win->skey + pInterval->interval - 1; + win->ekey = taosTimeAdd(win->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } } @@ -2727,7 +2806,7 @@ void filterRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInfo SSDataBlock* pBlock, bool ascQuery) { int32_t numOfRows = pBlock->info.rows; - int8_t *p = calloc(numOfRows, sizeof(int8_t)); + int8_t *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); bool all = true; #if 0 if (pRuntimeEnv->pTsBuf != NULL) { @@ -2763,7 +2842,7 @@ void filterRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInfo doCompactSDataBlock(pBlock, numOfRows, p); } - tfree(p); + taosMemoryFreeClear(p); } void filterColRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, bool ascQuery) { @@ -2774,7 +2853,7 @@ void filterColRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, if (pRuntimeEnv->pTsBuf != NULL) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); - p = calloc(numOfRows, sizeof(int8_t)); + p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); TSKEY* k = (TSKEY*) pColInfoData->pData; for (int32_t i = 0; i < numOfRows; ++i) { @@ -2810,7 +2889,7 @@ void filterColRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, } } - tfree(p); + taosMemoryFreeClear(p); } static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId); @@ -3331,7 +3410,7 @@ void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOf for(int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); - char* p = realloc(pColInfo->pData, newSize * pColInfo->info.bytes); + char* p = taosMemoryRealloc(pColInfo->pData, newSize * pColInfo->info.bytes); if (p != NULL) { pColInfo->pData = p; @@ -3519,7 +3598,7 @@ STableQueryInfo *createTableQueryInfo(void* buf, bool groupbyColumn, STimeWindow } STableQueryInfo* createTmpTableQueryInfo(STimeWindow win) { - STableQueryInfo* pTableQueryInfo = calloc(1, sizeof(STableQueryInfo)); + STableQueryInfo* pTableQueryInfo = taosMemoryCalloc(1, sizeof(STableQueryInfo)); // pTableQueryInfo->win = win; pTableQueryInfo->lastKey = win.skey; @@ -3528,7 +3607,7 @@ STableQueryInfo* createTmpTableQueryInfo(STimeWindow win) { int32_t initialSize = 16; int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize); if (code != TSDB_CODE_SUCCESS) { - tfree(pTableQueryInfo); + taosMemoryFreeClear(pTableQueryInfo); return NULL; } @@ -4346,7 +4425,7 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t assert(p->numOfDownstream == 0); } - p->pDownstream = calloc(1, num * POINTER_BYTES); + p->pDownstream = taosMemoryCalloc(1, num * POINTER_BYTES); if (p->pDownstream == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -4701,7 +4780,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo *pOperator, bool* newgroup) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0); int32_t len = (int32_t) tbufTell(&bw); - pColInfo->pData = malloc(len + sizeof(int32_t)); + pColInfo->pData = taosMemoryMalloc(len + sizeof(int32_t)); *(int32_t*) pColInfo->pData = len; memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len); @@ -4782,8 +4861,8 @@ int32_t loadRemoteDataCallback(void* param, const SDataBuf* pMsg, int32_t code) static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { assert(pMsgBody != NULL); - tfree(pMsgBody->msgInfo.pData); - tfree(pMsgBody); + taosMemoryFreeClear(pMsgBody->msgInfo.pData); + taosMemoryFreeClear(pMsgBody); } void qProcessFetchRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { @@ -4793,7 +4872,7 @@ void qProcessFetchRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { SDataBuf buf = {.len = pMsg->contLen, .pData = NULL}; if (pMsg->contLen > 0) { - buf.pData = calloc(1, pMsg->contLen); + buf.pData = taosMemoryCalloc(1, pMsg->contLen); if (buf.pData == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; pMsg->code = TSDB_CODE_OUT_OF_MEMORY; @@ -4810,7 +4889,7 @@ void qProcessFetchRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { static int32_t doSendFetchDataRequest(SExchangeInfo *pExchangeInfo, SExecTaskInfo *pTaskInfo, int32_t sourceIndex) { size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - SResFetchReq* pMsg = calloc(1, sizeof(SResFetchReq)); + SResFetchReq* pMsg = taosMemoryCalloc(1, sizeof(SResFetchReq)); if (NULL == pMsg) { pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return pTaskInfo->code; @@ -4828,9 +4907,9 @@ static int32_t doSendFetchDataRequest(SExchangeInfo *pExchangeInfo, SExecTaskInf pMsg->queryId = htobe64(pTaskInfo->id.queryId); // send the fetch remote task result reques - SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { - tfree(pMsg); + taosMemoryFreeClear(pMsg); qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return pTaskInfo->code; @@ -4868,7 +4947,7 @@ static int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* memcpy(pColInfoData->varmeta.offset, pStart, sizeof(int32_t)*numOfRows); pStart += sizeof(int32_t)*numOfRows; - pColInfoData->pData = malloc(colLen[i]); + pColInfoData->pData = taosMemoryMalloc(colLen[i]); } else { memcpy(pColInfoData->nullbitmap, pStart, BitmapLen(numOfRows)); pStart += BitmapLen(numOfRows); @@ -5158,8 +5237,8 @@ static SSDataBlock* doLoadRemoteData(SOperatorInfo *pOperator, bool* newgroup) { #if 0 _error: - tfree(pMsg); - tfree(pMsgSendInfo); + taosMemoryFreeClear(pMsg); + taosMemoryFreeClear(pMsgSendInfo); terrno = pTaskInfo->code; return NULL; @@ -5189,12 +5268,12 @@ static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo) { } SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { - SExchangeInfo* pInfo = calloc(1, sizeof(SExchangeInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SExchangeInfo* pInfo = taosMemoryCalloc(1, sizeof(SExchangeInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; } @@ -5263,14 +5342,14 @@ SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock destroyExchangeOperatorInfo(pInfo, numOfSources); } - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; return NULL; } SSDataBlock* createResultDataBlock(const SArray* pExprInfo) { - SSDataBlock* pResBlock = calloc(1, sizeof(SSDataBlock)); + SSDataBlock* pResBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); if (pResBlock == NULL) { return NULL; } @@ -5298,11 +5377,11 @@ SSDataBlock* createResultDataBlock(const SArray* pExprInfo) { SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo) { assert(repeatTime > 0); - STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; @@ -5327,7 +5406,7 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, } SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv) { - STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); + STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); pInfo->pTsdbReadHandle = pTsdbReadHandle; pInfo->times = 1; @@ -5337,7 +5416,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntim pInfo->prevGroupId = -1; pRuntimeEnv->enableGroupData = true; - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "TableSeqScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN; pOperator->blockingOptr = false; @@ -5351,7 +5430,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntim } SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv) { - STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); + STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); pInfo->pTsdbReadHandle = pTsdbReadHandle; pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); @@ -5362,7 +5441,7 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRunt infoData.info.colId = 0; taosArrayPush(pInfo->block.pDataBlock, &infoData); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "TableBlockInfoScanOperator"; // pOperator->operatorType = OP_TableBlockInfoScan; pOperator->blockingOptr = false; @@ -5375,11 +5454,11 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRunt } SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SSDataBlock* pResBlock, SArray* pColList, SArray* pTableIdList, SExecTaskInfo* pTaskInfo) { - SStreamBlockScanInfo* pInfo = calloc(1, sizeof(SStreamBlockScanInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; } @@ -5388,8 +5467,8 @@ SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SSDataBlock* tqReadHandleSetColIdList((STqReadHandle* )streamReadHandle, pColList); int32_t code = tqReadHandleSetTbUidList(streamReadHandle, pTableIdList); if (code != 0) { - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); return NULL; } @@ -5523,11 +5602,11 @@ static SSDataBlock* doSysTableScan(SOperatorInfo *pOperator, bool* newgroup) { strncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req); - char* buf1 = calloc(1, contLen); + char* buf1 = taosMemoryCalloc(1, contLen); tSerializeSRetrieveTableReq(buf1, contLen, &pInfo->req); // send the fetch remote task result reques - SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; @@ -5566,11 +5645,11 @@ static SSDataBlock* doSysTableScan(SOperatorInfo *pOperator, bool* newgroup) { SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataBlock* pResBlock, const SName* pName, SNode* pCondition, SEpSet epset, SArray* colList, SExecTaskInfo* pTaskInfo) { - SSysTableScanInfo* pInfo = calloc(1, sizeof(SSysTableScanInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SSysTableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SSysTableScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; } @@ -5753,7 +5832,7 @@ static void destroySlimitOperatorInfo(void* param, int32_t numOfOutput) { SSLimitOperatorInfo *pInfo = (SSLimitOperatorInfo*) param; taosArrayDestroy(pInfo->orderColumnList); pInfo->pRes = blockDataDestroy(pInfo->pRes); - tfree(pInfo->prevRow); + taosMemoryFreeClear(pInfo->prevRow); } static void assignExprInfo(SExprInfo* dst, const SExprInfo* src) { @@ -5762,7 +5841,7 @@ static void assignExprInfo(SExprInfo* dst, const SExprInfo* src) { *dst = *src; dst->pExpr = exprdup(src->pExpr); - dst->base.pParam = calloc(src->base.numOfParams, sizeof(SColumn)); + dst->base.pParam = taosMemoryCalloc(src->base.numOfParams, sizeof(SColumn)); memcpy(dst->base.pParam, src->base.pParam, sizeof(SColumn) * src->base.numOfParams); // memset(dst->base.param, 0, sizeof(SVariant) * tListLen(dst->base.param)); @@ -5774,7 +5853,7 @@ static void assignExprInfo(SExprInfo* dst, const SExprInfo* src) { static SExprInfo* exprArrayDup(SArray* pExprList) { size_t numOfOutput = taosArrayGetSize(pExprList); - SExprInfo* p = calloc(numOfOutput, sizeof(SExprInfo)); + SExprInfo* p = taosMemoryCalloc(numOfOutput, sizeof(SExprInfo)); for (int32_t i = 0; i < numOfOutput; ++i) { SExprInfo* pExpr = taosArrayGetP(pExprList, i); assignExprInfo(&p[i], pExpr); @@ -6008,11 +6087,11 @@ static SSDataBlock* doSortedMerge(SOperatorInfo *pOperator, bool* newgroup) { pInfo->pSortHandle = tsortCreateSortHandle(pInfo->orderInfo, pInfo->nullFirst, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage, p, pInfo->binfo.pRes->info.numOfCols, "GET_TASKID(pTaskInfo)"); - tfree(p); + taosMemoryFreeClear(p); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock); for(int32_t i = 0; i < pOperator->numOfDownstream; ++i) { - SGenericSource* ps = calloc(1, sizeof(SGenericSource)); + SGenericSource* ps = taosMemoryCalloc(1, sizeof(SGenericSource)); ps->param = pOperator->pDownstream[i]; tsortAddSource(pInfo->pSortHandle, ps); } @@ -6078,7 +6157,7 @@ static int32_t initGroupCol(SExprInfo* pExprInfo, int32_t numOfCols, SArray* pGr ASSERT(taosArrayGetSize(pGroupInfo) == taosArrayGetSize(plist)); - pInfo->groupVal = calloc(1, (POINTER_BYTES * numOfGroupCol + len)); + pInfo->groupVal = taosMemoryCalloc(1, (POINTER_BYTES * numOfGroupCol + len)); if (pInfo->groupVal == NULL) { taosArrayDestroy(plist); return TSDB_CODE_OUT_OF_MEMORY; @@ -6098,8 +6177,8 @@ static int32_t initGroupCol(SExprInfo* pExprInfo, int32_t numOfCols, SArray* pGr } SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pOrderVal, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo) { - SSortedMergeOperatorInfo* pInfo = calloc(1, sizeof(SSortedMergeOperatorInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SSortedMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortedMergeOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } @@ -6154,8 +6233,8 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t destroySortedMergeOperatorInfo(pInfo, num); } - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; } @@ -6176,10 +6255,10 @@ static SSDataBlock* doSort(SOperatorInfo *pOperator, bool* newgroup) { pInfo->pSortHandle = tsortCreateSortHandle(pInfo->orderInfo, pInfo->nullFirst, SORT_SINGLESOURCE_SORT, pInfo->bufPageSize, numOfBufPage, p, pInfo->pDataBlock->info.numOfCols, "GET_TASKID(pTaskInfo)"); - tfree(p); + taosMemoryFreeClear(p); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock); - SGenericSource* ps = calloc(1, sizeof(SGenericSource)); + SGenericSource* ps = taosMemoryCalloc(1, sizeof(SGenericSource)); ps->param = pOperator; tsortAddSource(pInfo->pSortHandle, ps); @@ -6194,11 +6273,11 @@ static SSDataBlock* doSort(SOperatorInfo *pOperator, bool* newgroup) { } SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SArray* pOrderVal, SExecTaskInfo* pTaskInfo) { - SOrderOperatorInfo* pInfo = calloc(1, sizeof(SOrderOperatorInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOrderOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SOrderOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; } @@ -6217,9 +6296,9 @@ SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx } if (pInfo->orderInfo == NULL || pInfo->pDataBlock == NULL) { - tfree(pOperator); + taosMemoryFreeClear(pOperator); destroyOrderOperatorInfo(pInfo, numOfCols); - tfree(pInfo); + taosMemoryFreeClear(pInfo); terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; @@ -6300,6 +6379,111 @@ static SSDataBlock* getAggregateResult(SOperatorInfo *pOperator, bool* newgroup) return (blockDataGetNumOfRows(pInfo->pRes) != 0)? pInfo->pRes:NULL; } +static void aggEncodeResultRow(SOperatorInfo* pOperator, char **result, int32_t *length) { + SAggOperatorInfo *pAggInfo = pOperator->info; + SAggSupporter *pSup = &pAggInfo->aggSup; + + int32_t size = taosHashGetSize(pSup->pResultRowHashTable); + size_t keyLen = POINTER_BYTES; // estimate the key length + int32_t totalSize = sizeof(int32_t) + size * (sizeof(int32_t) + keyLen + sizeof(int32_t) + pSup->resultRowSize); + *result = taosMemoryCalloc(1, totalSize); + if(*result == NULL){ + terrno = TSDB_CODE_OUT_OF_MEMORY; + return; + } + *(int32_t*)(*result) = size; + int32_t offset = sizeof(int32_t); + void *pIter = taosHashIterate(pSup->pResultRowHashTable, NULL); + while (pIter) { + void *key = taosHashGetKey(pIter, &keyLen); + SResultRow **p1 = (SResultRow **)pIter; + + // recalculate the result size + int32_t realTotalSize = offset + sizeof(int32_t) + keyLen + sizeof(int32_t) + pSup->resultRowSize; + if (realTotalSize > totalSize){ + char *tmp = taosMemoryRealloc(*result, realTotalSize); + if (tmp == NULL){ + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(*result); + *result = NULL; + return; + }else{ + *result = tmp; + } + } + // save key + *(int32_t*)(*result + offset) = keyLen; + offset += sizeof(int32_t); + memcpy(*result + offset, key, keyLen); + offset += keyLen; + + // save value + *(int32_t*)(*result + offset) = pSup->resultRowSize; + offset += sizeof(int32_t); + memcpy(*result + offset, *p1, pSup->resultRowSize); + offset += pSup->resultRowSize; + + pIter = taosHashIterate(pSup->pResultRowHashTable, pIter); + } + + if(length) { + *length = offset; + } + return; +} + +static bool aggDecodeResultRow(SOperatorInfo* pOperator, char *result, int32_t length) { + if (!result || length <= 0){ + return false; + } + + SAggOperatorInfo *pAggInfo = pOperator->info; + SAggSupporter *pSup = &pAggInfo->aggSup; + SOptrBasicInfo *pInfo = &pAggInfo->binfo; + + // int32_t size = taosHashGetSize(pSup->pResultRowHashTable); + int32_t count = *(int32_t*)(result); + + int32_t offset = sizeof(int32_t); + while(count-- > 0 && length > offset){ + int32_t keyLen = *(int32_t*)(result + offset); + offset += sizeof(int32_t); + + uint64_t tableGroupId = *(uint64_t *)(result + offset); + SResultRow *resultRow = getNewResultRow_rv(pSup->pResultBuf, tableGroupId, pSup->resultRowSize); + if (!resultRow){ + terrno = TSDB_CODE_TSC_INVALID_INPUT; + return false; + } + // add a new result set for a new group + taosHashPut(pSup->pResultRowHashTable, result + offset, keyLen, &resultRow, POINTER_BYTES); + + offset += keyLen; + int32_t valueLen = *(int32_t*)(result + offset); + if (valueLen != pSup->resultRowSize){ + terrno = TSDB_CODE_TSC_INVALID_INPUT; + return false; + } + offset += sizeof(int32_t); + int32_t pageId = resultRow->pageId; + int32_t pOffset = resultRow->offset; + memcpy(resultRow, result + offset, valueLen); + resultRow->pageId = pageId; + resultRow->offset = pOffset; + offset += valueLen; + + initResultRow(resultRow); + + pInfo->resultRowInfo.pPosition[pInfo->resultRowInfo.size++] = (SResultRowPosition) {.pageId = resultRow->pageId, .offset = resultRow->offset}; + } + + if (offset != length){ + terrno = TSDB_CODE_TSC_INVALID_INPUT; + return false; + } + return true; +} + static SSDataBlock* doMultiTableAggregate(SOperatorInfo *pOperator, bool* newgroup) { if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -6776,7 +6960,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI TSKEY* tsList = (TSKEY*)pTsColInfoData->pData; if (IS_REPEAT_SCAN(pRuntimeEnv) && !pInfo->reptScan) { pInfo->reptScan = true; - tfree(pInfo->prevData); + taosMemoryFreeClear(pInfo->prevData); } pInfo->numOfRows = 0; @@ -6786,7 +6970,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI continue; } if (pInfo->prevData == NULL) { - pInfo->prevData = malloc(bytes); + pInfo->prevData = taosMemoryMalloc(bytes); memcpy(pInfo->prevData, val, bytes); pInfo->numOfRows = 1; pInfo->curWindow.skey = tsList[j]; @@ -6969,7 +7153,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo *pOperator, bool* newgrou // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order); // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->binfo.pCtx, pOperator->numOfOutput); - doHashGroupbyAgg(pOperator, pInfo, pBlock); + doHashGroupbyAgg(pOperator, pBlock); } pOperator->status = OP_RES_TO_RETURN; @@ -6992,52 +7176,55 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo *pOperator, bool* newgrou return pInfo->binfo.pRes; } -static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo *pInfo, STaskRuntimeEnv* pRuntimeEnv, bool* newgroup) { +static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo *pInfo, SResultInfo* pResultInfo, bool* newgroup, SExecTaskInfo* pTaskInfo) { pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows; - int64_t ekey = Q_STATUS_EQUAL(pRuntimeEnv->status, TASK_COMPLETED)?pRuntimeEnv->pQueryAttr->window.ekey:pInfo->existNewGroupBlock->info.window.ekey; + + int64_t ekey = Q_STATUS_EQUAL(pTaskInfo->status, TASK_COMPLETED)? pTaskInfo->window.ekey:pInfo->existNewGroupBlock->info.window.ekey; taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo)); taosFillSetStartInfo(pInfo->pFillInfo, pInfo->existNewGroupBlock->info.rows, ekey); taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->existNewGroupBlock); - doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pRuntimeEnv->resultInfo.capacity, pInfo->p); + doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pResultInfo->capacity, pInfo->p); pInfo->existNewGroupBlock = NULL; *newgroup = true; } -static void doHandleRemainBlockFromNewGroup(SFillOperatorInfo *pInfo, STaskRuntimeEnv *pRuntimeEnv, bool *newgroup) { +static void doHandleRemainBlockFromNewGroup(SFillOperatorInfo *pInfo, SResultInfo *pResultInfo, bool *newgroup, SExecTaskInfo* pTaskInfo) { if (taosFillHasMoreResults(pInfo->pFillInfo)) { *newgroup = false; - doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, (int32_t)pRuntimeEnv->resultInfo.capacity, pInfo->p); - if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || (!pInfo->multigroupResult)) { + doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, (int32_t)pResultInfo->capacity, pInfo->p); + if (pInfo->pRes->info.rows > pResultInfo->threshold || (!pInfo->multigroupResult)) { return; } } // handle the cached new group data block if (pInfo->existNewGroupBlock) { - doHandleRemainBlockForNewGroupImpl(pInfo, pRuntimeEnv, newgroup); + doHandleRemainBlockForNewGroupImpl(pInfo, pResultInfo, newgroup, pTaskInfo); } } static SSDataBlock* doFill(SOperatorInfo *pOperator, bool* newgroup) { SFillOperatorInfo *pInfo = pOperator->info; - pInfo->pRes->info.rows = 0; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SResultInfo* pResultInfo = &pOperator->resultInfo; + blockDataCleanup(pInfo->pRes); if (pOperator->status == OP_EXEC_DONE) { return NULL; } - STaskRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; - doHandleRemainBlockFromNewGroup(pInfo, pRuntimeEnv, newgroup); - if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || (!pInfo->multigroupResult && pInfo->pRes->info.rows > 0)) { + doHandleRemainBlockFromNewGroup(pInfo, pResultInfo, newgroup, pTaskInfo); + if (pInfo->pRes->info.rows > pResultInfo->threshold || (!pInfo->multigroupResult && pInfo->pRes->info.rows > 0)) { return pInfo->pRes; } + SOperatorInfo* pDownstream = pOperator->pDownstream[0]; while(1) { - publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = pOperator->pDownstream[0]->getNextFn(pOperator->pDownstream[0], newgroup); - publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(pDownstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = pDownstream->getNextFn(pDownstream, newgroup); + publishOperatorProfEvent(pDownstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (*newgroup) { assert(pBlock != NULL); @@ -7049,7 +7236,7 @@ static SSDataBlock* doFill(SOperatorInfo *pOperator, bool* newgroup) { // Fill the previous group data block, before handle the data block of new group. // Close the fill operation for previous group data block - taosFillSetStartInfo(pInfo->pFillInfo, 0, pRuntimeEnv->pQueryAttr->window.ekey); + taosFillSetStartInfo(pInfo->pFillInfo, 0, pTaskInfo->window.ekey); } else { if (pBlock == NULL) { if (pInfo->totalInputRows == 0) { @@ -7057,7 +7244,7 @@ static SSDataBlock* doFill(SOperatorInfo *pOperator, bool* newgroup) { return NULL; } - taosFillSetStartInfo(pInfo->pFillInfo, 0, pRuntimeEnv->pQueryAttr->window.ekey); + taosFillSetStartInfo(pInfo->pFillInfo, 0, pTaskInfo->window.ekey); } else { pInfo->totalInputRows += pBlock->info.rows; taosFillSetStartInfo(pInfo->pFillInfo, pBlock->info.rows, pBlock->info.window.ekey); @@ -7065,25 +7252,24 @@ static SSDataBlock* doFill(SOperatorInfo *pOperator, bool* newgroup) { } } - doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pRuntimeEnv->resultInfo.capacity, pInfo->p); + doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pInfo->capacity, pInfo->p); // current group has no more result to return if (pInfo->pRes->info.rows > 0) { // 1. The result in current group not reach the threshold of output result, continue // 2. If multiple group results existing in one SSDataBlock is not allowed, return immediately - if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || pBlock == NULL || (!pInfo->multigroupResult)) { + if (pInfo->pRes->info.rows > pResultInfo->threshold || pBlock == NULL || (!pInfo->multigroupResult)) { return pInfo->pRes; } - doHandleRemainBlockFromNewGroup(pInfo, pRuntimeEnv, newgroup); - if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || pBlock == NULL) { + doHandleRemainBlockFromNewGroup(pInfo, pResultInfo, newgroup, pTaskInfo); + if (pInfo->pRes->info.rows > pOperator->resultInfo.threshold || pBlock == NULL) { return pInfo->pRes; } } else if (pInfo->existNewGroupBlock) { // try next group assert(pBlock != NULL); - doHandleRemainBlockForNewGroupImpl(pInfo, pRuntimeEnv, newgroup); - - if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold) { + doHandleRemainBlockForNewGroupImpl(pInfo, pResultInfo, newgroup, pTaskInfo); + if (pInfo->pRes->info.rows > pResultInfo->threshold) { return pInfo->pRes; } } else { @@ -7118,19 +7304,19 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { destroyOperatorInfo(pOperator->pDownstream[i]); } - tfree(pOperator->pDownstream); + taosMemoryFreeClear(pOperator->pDownstream); pOperator->numOfDownstream = 0; } - tfree(pOperator->info); - tfree(pOperator); + taosMemoryFreeClear(pOperator->info); + taosMemoryFreeClear(pOperator); } int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx *pCtx, int32_t numOfOutput, const char* pKey) { _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); - pAggSup->keyBuf = calloc(1, sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES); + pAggSup->keyBuf = taosMemoryCalloc(1, sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES); pAggSup->pResultRowHashTable = taosHashInit(10, hashFn, true, HASH_NO_LOCK); pAggSup->pResultRowListSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK); pAggSup->pResultRowArrayList = taosArrayInit(10, sizeof(SResultRowCell)); @@ -7149,7 +7335,7 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx *pCtx, int32_t n } static void cleanupAggSup(SAggSupporter* pAggSup) { - tfree(pAggSup->keyBuf); + taosMemoryFreeClear(pAggSup->keyBuf); taosHashCleanup(pAggSup->pResultRowHashTable); taosHashCleanup(pAggSup->pResultRowListSet); taosArrayDestroy(pAggSup->pResultRowArrayList); @@ -7166,7 +7352,7 @@ static int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, S } static STableQueryInfo* initTableQueryInfo(const STableGroupInfo* pTableGroupInfo) { - STableQueryInfo* pTableQueryInfo = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); + STableQueryInfo* pTableQueryInfo = taosMemoryCalloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); if (pTableQueryInfo == NULL) { return NULL; } @@ -7191,8 +7377,8 @@ static STableQueryInfo* initTableQueryInfo(const STableGroupInfo* pTableGroupInf SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) { - SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAggOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } @@ -7219,6 +7405,8 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->_openFn = doOpenAggregateOptr; pOperator->getNextFn = getAggregateResult; pOperator->closeFn = destroyAggOperatorInfo; + pOperator->encodeResultRow = aggEncodeResultRow; + pOperator->decodeResultRow = aggDecodeResultRow; code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -7228,8 +7416,8 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* return pOperator; _error: destroyAggOperatorInfo(pInfo, numOfCols); - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -7238,7 +7426,7 @@ static void doDestroyBasicInfo(SOptrBasicInfo* pInfo, int32_t numOfOutput) { assert(pInfo != NULL); destroySqlFunctionCtx(pInfo->pCtx, numOfOutput); - tfree(pInfo->rowCellInfoOffset); + taosMemoryFreeClear(pInfo->rowCellInfoOffset); cleanupResultRowInfo(&pInfo->resultRowInfo); pInfo->pRes = blockDataDestroy(pInfo->pRes); @@ -7252,7 +7440,7 @@ void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) { void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) { SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*) param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); - tfree(pInfo->prevData); + taosMemoryFreeClear(pInfo->prevData); } void destroyAggOperatorInfo(void* param, int32_t numOfOutput) { @@ -7275,13 +7463,15 @@ void destroySFillOperatorInfo(void* param, int32_t numOfOutput) { SFillOperatorInfo* pInfo = (SFillOperatorInfo*) param; pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo); pInfo->pRes = blockDataDestroy(pInfo->pRes); - tfree(pInfo->p); + taosMemoryFreeClear(pInfo->p); } void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput) { SGroupbyOperatorInfo* pInfo = (SGroupbyOperatorInfo*) param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); - tfree(pInfo->prevData); + taosMemoryFreeClear(pInfo->keyBuf); + taosArrayDestroy(pInfo->pGroupCols); + taosArrayDestroy(pInfo->pGroupColVals); } static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) { @@ -7309,7 +7499,7 @@ static void destroyConditionOperatorInfo(void* param, int32_t numOfOutput) { static void destroyDistinctOperatorInfo(void* param, int32_t numOfOutput) { SDistinctOperatorInfo* pInfo = (SDistinctOperatorInfo*) param; taosHashCleanup(pInfo->pSet); - tfree(pInfo->buf); + taosMemoryFreeClear(pInfo->buf); taosArrayDestroy(pInfo->pDistinctDataInfo); pInfo->pRes = blockDataDestroy(pInfo->pRes); } @@ -7336,7 +7526,7 @@ void destroyExchangeOperatorInfo(void* param, int32_t numOfOutput) { } SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) { - SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); + SAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAggOperatorInfo)); int32_t numOfRows = 1; int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, numOfRows, pResBlock, pTaskInfo->id.str); @@ -7348,7 +7538,7 @@ SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SExprI size_t tableGroup = taosArrayGetSize(pTableGroupInfo->pGroupList); initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)tableGroup); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "MultiTableAggregate"; // pOperator->operatorType = OP_MultiTableAggregate; pOperator->blockingOptr = true; @@ -7372,8 +7562,8 @@ _error: } SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t num, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo) { - SProjectOperatorInfo* pInfo = calloc(1, sizeof(SProjectOperatorInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SProjectOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SProjectOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } @@ -7413,7 +7603,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int32_t* numOfFilterCols) { #if 0 - SColumnInfo* pCols = calloc(numOfOutput, sizeof(SColumnInfo)); + SColumnInfo* pCols = taosMemoryCalloc(numOfOutput, sizeof(SColumnInfo)); int32_t numOfFilter = 0; for(int32_t i = 0; i < numOfOutput; ++i) { @@ -7427,7 +7617,7 @@ SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int3 pCols[i].flist.numOfFilters = pExpr[i].base.flist.numOfFilters; if (pCols[i].flist.numOfFilters != 0) { - pCols[i].flist.filterInfo = calloc(pCols[i].flist.numOfFilters, sizeof(SColumnFilterInfo)); + pCols[i].flist.filterInfo = taosMemoryCalloc(pCols[i].flist.numOfFilters, sizeof(SColumnFilterInfo)); memcpy(pCols[i].flist.filterInfo, pExpr[i].base.flist.filterInfo, pCols[i].flist.numOfFilters * sizeof(SColumnFilterInfo)); } else { // avoid runtime error @@ -7444,10 +7634,9 @@ SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int3 return 0; } -SOperatorInfo* createLimitOperatorInfo(SOperatorInfo* downstream, int32_t numOfDownstream, SLimit* pLimit, SExecTaskInfo* pTaskInfo) { - ASSERT(numOfDownstream == 1); - SLimitOperatorInfo* pInfo = calloc(1, sizeof(SLimitOperatorInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); +SOperatorInfo* createLimitOperatorInfo(SOperatorInfo* downstream, SLimit* pLimit, SExecTaskInfo* pTaskInfo) { + SLimitOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SLimitOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } @@ -7467,22 +7656,22 @@ SOperatorInfo* createLimitOperatorInfo(SOperatorInfo* downstream, int32_t numOfD return pOperator; _error: - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo) { - STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } pInfo->order = TSDB_ORDER_ASC; - pInfo->precision = TSDB_TIME_PRECISION_MICRO; + pInfo->precision = TSDB_TIME_PRECISION_MILLI; pInfo->win = pTaskInfo->window; pInfo->interval = *pInterval; @@ -7519,20 +7708,20 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* _error: destroyIntervalOperatorInfo(pInfo, numOfCols); - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); pTaskInfo->code = code; return NULL; } SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { - STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); + STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); +// pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pResultInfo->capacity); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "AllTimeIntervalAggOperator"; // pOperator->operatorType = OP_AllTimeWindow; @@ -7550,14 +7739,14 @@ SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S } SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { - SStateWindowOperatorInfo* pInfo = calloc(1, sizeof(SStateWindowOperatorInfo)); + SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo)); pInfo->colIndex = -1; pInfo->reptScan = false; pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); +// pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pResultInfo->capacity); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "StateWindowOperator"; // pOperator->operatorType = OP_StateWindow; pOperator->blockingOptr = true; @@ -7574,8 +7763,8 @@ SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOper } SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo) { - SSessionAggOperatorInfo* pInfo = calloc(1, sizeof(SSessionAggOperatorInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } @@ -7609,20 +7798,20 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo destroySWindowOperatorInfo(pInfo, numOfCols); } - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); pTaskInfo->code = code; return NULL; } SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { - STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); + STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); +// pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pResultInfo->capacity); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "MultiTableTimeIntervalOperator"; // pOperator->operatorType = OP_MultiTableTimeInterval; pOperator->blockingOptr = true; @@ -7640,13 +7829,13 @@ SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntim } SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { - STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); + STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); +// pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pResultInfo->capacity); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "AllMultiTableTimeIntervalOperator"; // pOperator->operatorType = OP_AllMultiTableTimeInterval; pOperator->blockingOptr = true; @@ -7664,19 +7853,56 @@ SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRun return pOperator; } +static int32_t initGroupOptrInfo(SGroupbyOperatorInfo *pInfo, SArray* pGroupColList) { + pInfo->pGroupColVals = taosArrayInit(4, sizeof(SGroupKeys)); + if (pInfo->pGroupColVals == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t numOfGroupCols = taosArrayGetSize(pGroupColList); + for(int32_t i = 0; i < numOfGroupCols; ++i) { + SColumn* pCol = taosArrayGet(pGroupColList, i); + pInfo->groupKeyLen += pCol->bytes; + + struct SGroupKeys key = {0}; + key.bytes = pCol->bytes; + key.type = pCol->type; + key.isNull = false; + key.pData = taosMemoryCalloc(1, pCol->bytes); + if (key.pData == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + taosArrayPush(pInfo->pGroupColVals, &key); + } + + int32_t nullFlagSize = sizeof(int8_t) * numOfGroupCols; + pInfo->keyBuf = taosMemoryCalloc(1, pInfo->groupKeyLen + nullFlagSize); + + if (pInfo->keyBuf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + return TSDB_CODE_SUCCESS; +} + SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SArray* pGroupColList, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) { - SGroupbyOperatorInfo* pInfo = calloc(1, sizeof(SGroupbyOperatorInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SGroupbyOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SGroupbyOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } pInfo->pGroupCols = pGroupColList; initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, 4096, pResultBlock, pTaskInfo->id.str); - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); + int32_t code = initGroupOptrInfo(pInfo, pGroupColList); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + pOperator->name = "GroupbyAggOperator"; pOperator->blockingOptr = true; pOperator->status = OP_NOT_OPENED; @@ -7688,61 +7914,85 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx pOperator->getNextFn = hashGroupbyAggregate; pOperator->closeFn = destroyGroupbyOperatorInfo; - int32_t code = appendDownstream(pOperator, &downstream, 1); + code = appendDownstream(pOperator, &downstream, 1); return pOperator; _error: - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); return NULL; } -SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult) { - SFillOperatorInfo* pInfo = calloc(1, sizeof(SFillOperatorInfo)); - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - pInfo->multigroupResult = multigroupResult; +static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t numOfCols, int64_t* fillVal, + STimeWindow win, int32_t capacity, const char* id, SInterval* pInterval, int32_t fillType) { + struct SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, (int64_t*)fillVal); - { - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - struct SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfOutput, pQueryAttr->fillVal); - STimeWindow w = TSWINDOW_INITIALIZER; + TSKEY sk = TMIN(win.skey, win.ekey); + TSKEY ek = TMAX(win.skey, win.ekey); + + // TODO set correct time precision + STimeWindow w = TSWINDOW_INITIALIZER; + getAlignQueryTimeWindow(pInterval, TSDB_TIME_PRECISION_MILLI, win.skey, sk, ek, &w); - TSKEY sk = TMIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); - TSKEY ek = TMAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); -// getAlignQueryTimeWindow(pQueryAttr, pQueryAttr->window.skey, sk, ek, &w); + int32_t order = TSDB_ORDER_ASC; + pInfo->pFillInfo = + taosCreateFillInfo(order, w.skey, 0, capacity, numOfCols, pInterval->sliding, + pInterval->slidingUnit, (int8_t)pInterval->precision, fillType, pColInfo, id); - pInfo->pFillInfo = - taosCreateFillInfo(pQueryAttr->order.order, w.skey, 0, (int32_t)pRuntimeEnv->resultInfo.capacity, numOfOutput, - pQueryAttr->interval.sliding, pQueryAttr->interval.slidingUnit, - (int8_t)pQueryAttr->precision, pQueryAttr->fillType, pColInfo, pRuntimeEnv->qinfo); + pInfo->p = taosMemoryCalloc(numOfCols, POINTER_BYTES); - pInfo->p = calloc(numOfOutput, POINTER_BYTES); + if (pInfo->pFillInfo == NULL || pInfo->p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } else { + return TSDB_CODE_SUCCESS; } +} - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); +SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SInterval* pInterval, SSDataBlock* pResBlock, + int32_t fillType, char* fillVal, bool multigroupResult, SExecTaskInfo* pTaskInfo) { + SFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SFillOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + + pInfo->pRes = pResBlock; + pInfo->multigroupResult = multigroupResult; + pInfo->intervalInfo = *pInterval; + + SResultInfo* pResultInfo = &pOperator->resultInfo; + int32_t code = initFillInfo(pInfo, pExpr, numOfCols, (int64_t*) fillVal, pTaskInfo->window, pResultInfo->capacity, pTaskInfo->id.str, pInterval, fillType); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } pOperator->name = "FillOperator"; pOperator->blockingOptr = false; pOperator->status = OP_NOT_OPENED; // pOperator->operatorType = OP_Fill; pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; + pOperator->numOfOutput = numOfCols; pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->getNextFn = doFill; - pOperator->closeFn = destroySFillOperatorInfo; + pOperator->_openFn = operatorDummyOpenFn; + pOperator->getNextFn = doFill; + pOperator->pTaskInfo = pTaskInfo; - int32_t code = appendDownstream(pOperator, &downstream, 1); + pOperator->closeFn = destroySFillOperatorInfo; + + code = appendDownstream(pOperator, &downstream, 1); return pOperator; + + _error: + taosMemoryFreeClear(pOperator); + taosMemoryFreeClear(pInfo); + return NULL; } SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, void* pMerger, bool multigroupResult) { - SSLimitOperatorInfo* pInfo = calloc(1, sizeof(SSLimitOperatorInfo)); + SSLimitOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSLimitOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); // pInfo->orderColumnList = getResultGroupCheckColumns(pQueryAttr); // pInfo->slimit = pQueryAttr->slimit; // pInfo->limit = pQueryAttr->limit; -// pInfo->capacity = pRuntimeEnv->resultInfo.capacity; +// pInfo->capacity = pResultInfo->capacity; // pInfo->threshold = (int64_t)(pInfo->capacity * 0.8); // pInfo->currentOffset = pQueryAttr->limit.offset; // pInfo->currentGroupOffset = pQueryAttr->slimit.offset; @@ -7755,7 +8005,7 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI } int32_t numOfCols = (pInfo->orderColumnList != NULL)? (int32_t) taosArrayGetSize(pInfo->orderColumnList):0; - pInfo->prevRow = calloc(1, (POINTER_BYTES * numOfCols + len)); + pInfo->prevRow = taosMemoryCalloc(1, (POINTER_BYTES * numOfCols + len)); int32_t offset = POINTER_BYTES * numOfCols; for(int32_t i = 0; i < numOfCols; ++i) { @@ -7765,9 +8015,8 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI offset += pExpr[index->colIndex].base.resSchema.bytes; } - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pOperator->resultInfo.capacity); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "SLimitOperator"; // pOperator->operatorType = OP_SLimit; @@ -7790,7 +8039,7 @@ static SSDataBlock* doTagScan(SOperatorInfo *pOperator, bool* newgroup) { } STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - int32_t maxNumOfTables = (int32_t)pRuntimeEnv->resultInfo.capacity; + int32_t maxNumOfTables = (int32_t)pResultInfo->capacity; STagScanInfo *pInfo = pOperator->info; SSDataBlock *pRes = pInfo->pRes; @@ -7915,8 +8164,8 @@ static SSDataBlock* doTagScan(SOperatorInfo *pOperator, bool* newgroup) { } SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput) { - STagScanInfo* pInfo = calloc(1, sizeof(STagScanInfo)); - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo)); +// pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pResultInfo->capacity); size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); assert(numOfGroup == 0 || numOfGroup == 1); @@ -7924,7 +8173,7 @@ SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo pInfo->totalTables = pRuntimeEnv->tableqinfoGroupInfo.numOfTables; pInfo->curPos = 0; - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "SeqTableTagScan"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; pOperator->blockingOptr = false; @@ -7958,7 +8207,7 @@ static bool initMultiDistinctInfo(SDistinctOperatorInfo *pInfo, SOperatorInfo* p } } pInfo->totalBytes += (int32_t)strlen(MULTI_KEY_DELIM) * (pOperator->numOfOutput); - pInfo->buf = calloc(1, pInfo->totalBytes); + pInfo->buf = taosMemoryCalloc(1, pInfo->totalBytes); return taosArrayGetSize(pInfo->pDistinctDataInfo) == pOperator->numOfOutput ? true : false; } @@ -8016,7 +8265,7 @@ static SSDataBlock* hashDistinct(SOperatorInfo *pOperator, bool* newgroup) { for (int i = 0; i < taosArrayGetSize(pRes->pDataBlock); i++) { SColumnInfoData* pResultColInfoData = taosArrayGet(pRes->pDataBlock, i); SDistinctDataInfo* pDistDataInfo = taosArrayGet(pInfo->pDistinctDataInfo, i); - char* tmp = realloc(pResultColInfoData->pData, newSize * pDistDataInfo->bytes); + char* tmp = taosMemoryRealloc(pResultColInfoData->pData, newSize * pDistDataInfo->bytes); if (tmp == NULL) { return NULL; } else { @@ -8052,7 +8301,7 @@ static SSDataBlock* hashDistinct(SOperatorInfo *pOperator, bool* newgroup) { } SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { - SDistinctOperatorInfo* pInfo = calloc(1, sizeof(SDistinctOperatorInfo)); + SDistinctOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SDistinctOperatorInfo)); pInfo->totalBytes = 0; pInfo->buf = NULL; pInfo->threshold = tsMaxNumOfDistinctResults; // distinct result threshold @@ -8062,7 +8311,7 @@ SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperato pInfo->pRes = createOutputBuf(pExpr, numOfOutput, (int32_t) pInfo->outputCapacity); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "DistinctOperator"; pOperator->blockingOptr = false; pOperator->status = OP_NOT_OPENED; @@ -8127,7 +8376,7 @@ static int32_t deserializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t if (pColFilter->filterstr) { pColFilter->len = htobe64(pFilterMsg->len); - pColFilter->pz = (int64_t)calloc(1, (size_t)(pColFilter->len + 1 * TSDB_NCHAR_SIZE)); // note: null-terminator + pColFilter->pz = (int64_t)taosMemoryCalloc(1, (size_t)(pColFilter->len + 1 * TSDB_NCHAR_SIZE)); // note: null-terminator if (pColFilter->pz == 0) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -8166,7 +8415,7 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* } *numOfExprs = numOfFuncs + numOfGroupKeys; - SExprInfo* pExprs = calloc(*numOfExprs, sizeof(SExprInfo)); + SExprInfo* pExprs = taosMemoryCalloc(*numOfExprs, sizeof(SExprInfo)); for(int32_t i = 0; i < (*numOfExprs); ++i) { STargetNode* pTargetNode = NULL; @@ -8178,14 +8427,14 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* SExprInfo* pExp = &pExprs[pTargetNode->slotId]; - pExp->pExpr = calloc(1, sizeof(tExprNode)); + pExp->pExpr = taosMemoryCalloc(1, sizeof(tExprNode)); pExp->pExpr->_function.num = 1; pExp->pExpr->_function.functionId = -1; - pExp->base.pParam = calloc(1, sizeof(SFunctParam)); + pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam)); pExp->base.numOfParams = 1; - pExp->base.pParam[0].pCol = calloc(1, sizeof(SColumn)); + pExp->base.pParam[0].pCol = taosMemoryCalloc(1, sizeof(SColumn)); SColumn* pCol = pExp->base.pParam[0].pCol; // it is a project query, or group by column @@ -8229,20 +8478,20 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* } static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId) { - SExecTaskInfo* pTaskInfo = calloc(1, sizeof(SExecTaskInfo)); + SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo)); setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTaskInfo->cost.created = taosGetTimestampMs(); pTaskInfo->id.queryId = queryId; - char* p = calloc(1, 128); + char* p = taosMemoryCalloc(1, 128); snprintf(p, 128, "TID:0x%"PRIx64" QID:0x%"PRIx64, taskId, queryId); pTaskInfo->id.str = strdup(p); return pTaskInfo; } -static tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId); +static tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, STableGroupInfo *pTableGroupInfo, uint64_t queryId, uint64_t taskId); static int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId); static SArray* extractTableIdList(const STableGroupInfo* pTableGroupInfo); @@ -8250,20 +8499,13 @@ static SArray* extractScanColumnId(SNodeList* pNodeList); static SArray* extractColumnInfo(SNodeList* pNodeList); SOperatorInfo* doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId, STableGroupInfo* pTableGroupInfo) { -// if (nodeType(pPhyNode) == QUERY_NODE_PHYSICAL_PLAN_PROJECT) { // ignore the project node -// pPhyNode = nodesListGetNode(pPhyNode->pChildren, 0); -// } - if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) { if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pPhyNode)) { SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; size_t numOfCols = LIST_LENGTH(pScanPhyNode->pScanCols); - tsdbReaderT pDataReader = doCreateDataReader((STableScanPhysiNode*)pPhyNode, pHandle, (uint64_t)queryId, taskId); - - int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId); - return createTableScanOperatorInfo(pDataReader, pScanPhyNode->order, numOfCols, pScanPhyNode->count, - pScanPhyNode->reverse, pTaskInfo); + tsdbReaderT pDataReader = doCreateDataReader((STableScanPhysiNode*)pPhyNode, pHandle, pTableGroupInfo, (uint64_t)queryId, taskId); + return createTableScanOperatorInfo(pDataReader, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pScanPhyNode->reverse, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == nodeType(pPhyNode)) { SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pPhyNode; SSDataBlock* pResBlock = createOutputBuf_rv1(pExchange->node.pOutputDataBlockDesc); @@ -8271,10 +8513,8 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTa } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == nodeType(pPhyNode)) { SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. - STableGroupInfo groupInfo = {0}; - - int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, &groupInfo, queryId, taskId); - SArray* tableIdList = extractTableIdList(&groupInfo); + int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId); + SArray* tableIdList = extractTableIdList(pTableGroupInfo); SSDataBlock* pResBlock = createOutputBuf_rv1(pScanPhyNode->node.pOutputDataBlockDesc); SArray* colList = extractScanColumnId(pScanPhyNode->pScanCols); @@ -8342,10 +8582,12 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTa SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; int32_t num = 0; - SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->pFuncs, NULL, &num); + SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num); SSDataBlock* pResBlock = createOutputBuf_rv1(pPhyNode->pOutputDataBlockDesc); - SInterval interval = {.interval = pIntervalPhyNode->interval, .sliding = pIntervalPhyNode->sliding, .intervalUnit = 'a', .slidingUnit = 'a'}; + SInterval interval = {.interval = pIntervalPhyNode->interval, .sliding = pIntervalPhyNode->sliding, + .intervalUnit = pIntervalPhyNode->intervalUnit, + .slidingUnit = pIntervalPhyNode->slidingUnit, .offset = pIntervalPhyNode->offset}; return createIntervalOperatorInfo(op, pExprInfo, num, pResBlock, &interval, pTableGroupInfo, pTaskInfo); } } /*else if (pPhyNode->info.type == OP_MultiTableAggregate) { @@ -8365,7 +8607,7 @@ static tsdbReaderT createDataReaderImpl(STableScanPhysiNode* pTableScanNode, STa cond.order = pTableScanNode->scan.order; cond.numOfCols = LIST_LENGTH(pTableScanNode->scan.pScanCols); - cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); + cond.colList = taosMemoryCalloc(cond.numOfCols, sizeof(SColumnInfo)); if (cond.colList == NULL) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; @@ -8465,22 +8707,20 @@ SArray* extractTableIdList(const STableGroupInfo* pTableGroupInfo) { return tableIdList; } -tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId) { - STableGroupInfo groupInfo = {0}; - +tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, STableGroupInfo *pTableGroupInfo, uint64_t queryId, uint64_t taskId) { uint64_t uid = pTableScanNode->scan.uid; - int32_t code = doCreateTableGroup(pHandle->meta, pTableScanNode->scan.tableType, uid, &groupInfo, queryId, taskId); + int32_t code = doCreateTableGroup(pHandle->meta, pTableScanNode->scan.tableType, uid, pTableGroupInfo, queryId, taskId); if (code != TSDB_CODE_SUCCESS) { goto _error; } - if (groupInfo.numOfTables == 0) { + if (pTableGroupInfo->numOfTables == 0) { code = 0; qDebug("no table qualified for query, TID:0x%"PRIx64", QID:0x%"PRIx64, taskId, queryId); goto _error; } - return createDataReaderImpl(pTableScanNode, &groupInfo, pHandle->reader, queryId, taskId); + return createDataReaderImpl(pTableScanNode, pTableGroupInfo, pHandle->reader, queryId, taskId); _error: terrno = code; @@ -8512,7 +8752,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead return code; _complete: - tfree(*pTaskInfo); + taosMemoryFreeClear(*pTaskInfo); terrno = code; return code; @@ -8523,7 +8763,7 @@ int32_t cloneExprFilterInfo(SColumnFilterInfo **dst, SColumnFilterInfo* src, int return TSDB_CODE_SUCCESS; } - *dst = calloc(filterNum, sizeof(*src)); + *dst = taosMemoryCalloc(filterNum, sizeof(*src)); if (*dst == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -8532,11 +8772,11 @@ int32_t cloneExprFilterInfo(SColumnFilterInfo **dst, SColumnFilterInfo* src, int for (int32_t i = 0; i < filterNum; i++) { if ((*dst)[i].filterstr && dst[i]->len > 0) { - void *pz = calloc(1, (size_t)(*dst)[i].len + 1); + void *pz = taosMemoryCalloc(1, (size_t)(*dst)[i].len + 1); if (pz == NULL) { if (i == 0) { - free(*dst); + taosMemoryFree(*dst); } else { freeColumnFilterInfo(*dst, i); } @@ -8596,7 +8836,7 @@ int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters) { } //int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId) { -// *pFilterInfo = calloc(1, sizeof(SSingleColumnFilterInfo) * numOfFilterCols); +// *pFilterInfo = taosMemoryCalloc(1, sizeof(SSingleColumnFilterInfo) * numOfFilterCols); // if (*pFilterInfo == NULL) { // return TSDB_CODE_QRY_OUT_OF_MEMORY; // } @@ -8609,7 +8849,7 @@ int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters) { // pFilter->info = pCols[i]; // // pFilter->numOfFilters = pCols[i].flist.numOfFilters; -// pFilter->pFilters = calloc(pFilter->numOfFilters, sizeof(SColumnFilterElem)); +// pFilter->pFilters = taosMemoryCalloc(pFilter->numOfFilters, sizeof(SColumnFilterElem)); // if (pFilter->pFilters == NULL) { // return TSDB_CODE_QRY_OUT_OF_MEMORY; // } @@ -8651,11 +8891,11 @@ void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFil // if (pFilterInfo[i].pFilters->filterInfo.lowerRelOptr == TSDB_RELATION_IN) { // taosHashCleanup((SHashObj *)(pFilterInfo[i].pFilters->q)); // } -// tfree(pFilterInfo[i].pFilters); +// taosMemoryFreeClear(pFilterInfo[i].pFilters); // } // } // -// tfree(pFilterInfo); +// taosMemoryFreeClear(pFilterInfo); return NULL; } @@ -8715,7 +8955,7 @@ static void doUpdateExprColumnIndex(STaskAttr *pQueryAttr) { } } -void setResultBufSize(STaskAttr* pQueryAttr, SRspResultInfo* pResultInfo) { +void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo) { const int32_t DEFAULT_RESULT_MSG_SIZE = 1024 * (1024 + 512); // the minimum number of rows for projection query @@ -8736,7 +8976,7 @@ void setResultBufSize(STaskAttr* pQueryAttr, SRspResultInfo* pResultInfo) { } pResultInfo->threshold = (int32_t)(pResultInfo->capacity * THRESHOLD_RATIO); - pResultInfo->total = 0; + pResultInfo->totalRows = 0; } //TODO refactor @@ -8747,11 +8987,11 @@ void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters) { for (int32_t i = 0; i < numOfFilters; i++) { if (pFilter[i].filterstr && pFilter[i].pz) { - free((void*)(pFilter[i].pz)); + taosMemoryFree((void*)(pFilter[i].pz)); } } - free(pFilter); + taosMemoryFree(pFilter); } static void doDestroyTableQueryInfo(STableGroupInfo* pTableqinfoGroupInfo) { @@ -8785,9 +9025,9 @@ void doDestroyTask(SExecTaskInfo *pTaskInfo) { // taosArrayDestroy(pTaskInfo->summary.queryProfEvents); // taosHashCleanup(pTaskInfo->summary.operatorProfResults); - tfree(pTaskInfo->sql); - tfree(pTaskInfo->id.str); - tfree(pTaskInfo); + taosMemoryFreeClear(pTaskInfo->sql); + taosMemoryFreeClear(pTaskInfo->id.str); + taosMemoryFreeClear(pTaskInfo); } static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes) { diff --git a/source/libs/executor/src/tlinearhash.c b/source/libs/executor/src/tlinearhash.c index 28319469ccc0ad039490de31cea5d55a6350198e..1b55467c8587e2b2132bae8b3f549bb7a7c39035 100644 --- a/source/libs/executor/src/tlinearhash.c +++ b/source/libs/executor/src/tlinearhash.c @@ -209,7 +209,7 @@ static int32_t doAddNewBucket(SLHashObj* pHashObj) { newLen += 4; } - char* p = realloc(pHashObj->pBucket, POINTER_BYTES * newLen); + char* p = taosMemoryRealloc(pHashObj->pBucket, POINTER_BYTES * newLen); if (p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -219,7 +219,7 @@ static int32_t doAddNewBucket(SLHashObj* pHashObj) { pHashObj->numOfAlloc = newLen; } - SLHashBucket* pBucket = calloc(1, sizeof(SLHashBucket)); + SLHashBucket* pBucket = taosMemoryCalloc(1, sizeof(SLHashBucket)); pHashObj->pBucket[pHashObj->numOfBuckets] = pBucket; pBucket->pPageIdList = taosArrayInit(2, sizeof(int32_t)); @@ -241,7 +241,7 @@ static int32_t doAddNewBucket(SLHashObj* pHashObj) { } SLHashObj* tHashInit(int32_t inMemPages, int32_t pageSize, _hash_fn_t fn, int32_t numOfTuplePerPage) { - SLHashObj* pHashObj = calloc(1, sizeof(SLHashObj)); + SLHashObj* pHashObj = taosMemoryCalloc(1, sizeof(SLHashObj)); if (pHashObj == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -265,12 +265,12 @@ SLHashObj* tHashInit(int32_t inMemPages, int32_t pageSize, _hash_fn_t fn, int32_ pHashObj->tuplesPerPage = numOfTuplePerPage; pHashObj->numOfAlloc = 4; // initial allocated array list - pHashObj->pBucket = calloc(pHashObj->numOfAlloc, POINTER_BYTES); + pHashObj->pBucket = taosMemoryCalloc(pHashObj->numOfAlloc, POINTER_BYTES); code = doAddNewBucket(pHashObj); if (code != TSDB_CODE_SUCCESS) { destroyDiskbasedBuf(pHashObj->pBuf); - tfree(pHashObj); + taosMemoryFreeClear(pHashObj); terrno = code; return NULL; } @@ -282,11 +282,11 @@ void* tHashCleanup(SLHashObj* pHashObj) { destroyDiskbasedBuf(pHashObj->pBuf); for(int32_t i = 0; i < pHashObj->numOfBuckets; ++i) { taosArrayDestroy(pHashObj->pBucket[i]->pPageIdList); - tfree(pHashObj->pBucket[i]); + taosMemoryFreeClear(pHashObj->pBucket[i]); } - tfree(pHashObj->pBucket); - tfree(pHashObj); + taosMemoryFreeClear(pHashObj->pBucket); + taosMemoryFreeClear(pHashObj); return NULL; } diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c index 65067f58e38338c76e98c467cd27aa65e64cb631..981da0415e63684841099f24729891f0e8ccb5a1 100644 --- a/source/libs/executor/src/tsimplehash.c +++ b/source/libs/executor/src/tsimplehash.c @@ -29,7 +29,7 @@ #define FREE_HASH_NODE(_n) \ do { \ - tfree(_n); \ + taosMemoryFreeClear(_n); \ } while (0); typedef struct SHNode { @@ -62,7 +62,7 @@ SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t capacity = 4; } - SSHashObj* pHashObj = (SSHashObj*) calloc(1, sizeof(SSHashObj)); + SSHashObj* pHashObj = (SSHashObj*) taosMemoryCalloc(1, sizeof(SSHashObj)); if (pHashObj == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -78,9 +78,9 @@ SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t pHashObj->keyLen = keyLen; pHashObj->dataLen = dataLen; - pHashObj->hashList = (SHNode **)calloc(pHashObj->capacity, sizeof(void *)); + pHashObj->hashList = (SHNode **)taosMemoryCalloc(pHashObj->capacity, sizeof(void *)); if (pHashObj->hashList == NULL) { - free(pHashObj); + taosMemoryFree(pHashObj); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -95,7 +95,7 @@ int32_t tSimpleHashGetSize(const SSHashObj *pHashObj) { } static SHNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { - SHNode *pNewNode = malloc(sizeof(SHNode) + keyLen + dsize); + SHNode *pNewNode = taosMemoryMalloc(sizeof(SHNode) + keyLen + dsize); if (pNewNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -120,7 +120,7 @@ void taosHashTableResize(SSHashObj *pHashObj) { } int64_t st = taosGetTimestampUs(); - void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newCapacity); + void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity); if (pNewEntryList == NULL) { // qWarn("hash resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); return; @@ -287,7 +287,7 @@ void tSimpleHashCleanup(SSHashObj *pHashObj) { } tSimpleHashClear(pHashObj); - tfree(pHashObj->hashList); + taosMemoryFreeClear(pHashObj->hashList); } size_t tSimpleHashGetMemSize(const SSHashObj *pHashObj) { diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index d424599758baf87eefe85b6bcc28abb374247fbf..85ba462c9a87de3af56089dd09fc384f7ab3af19 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -65,7 +65,7 @@ typedef struct SSortHandle { static int32_t msortComparFn(const void *pLeft, const void *pRight, void *param); static SSDataBlock* createDataBlock_rv(SSchema* pSchema, int32_t numOfCols) { - SSDataBlock* pBlock = calloc(1, sizeof(SSDataBlock)); + SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); pBlock->info.numOfCols = numOfCols; @@ -91,7 +91,7 @@ static SSDataBlock* createDataBlock_rv(SSchema* pSchema, int32_t numOfCols) { * @return */ SSortHandle* tsortCreateSortHandle(SArray* pOrderInfo, bool nullFirst, int32_t type, int32_t pageSize, int32_t numOfPages, SSchema* pSchema, int32_t numOfCols, const char* idstr) { - SSortHandle* pSortHandle = calloc(1, sizeof(SSortHandle)); + SSortHandle* pSortHandle = taosMemoryCalloc(1, sizeof(SSortHandle)); pSortHandle->type = type; pSortHandle->pageSize = pageSize; @@ -118,8 +118,8 @@ void tsortDestroySortHandle(SSortHandle* pSortHandle) { } destroyDiskbasedBuf(pSortHandle->pBuf); - tfree(pSortHandle->idStr); - tfree(pSortHandle); + taosMemoryFreeClear(pSortHandle->idStr); + taosMemoryFreeClear(pSortHandle); } int32_t tsortAddSource(SSortHandle* pSortHandle, void* pSource) { @@ -127,7 +127,7 @@ int32_t tsortAddSource(SSortHandle* pSortHandle, void* pSource) { } static int32_t doAddNewExternalMemSource(SDiskbasedBuf *pBuf, SArray* pAllSources, SSDataBlock* pBlock, int32_t* sourceId) { - SExternalMemSource* pSource = calloc(1, sizeof(SExternalMemSource)); + SExternalMemSource* pSource = taosMemoryCalloc(1, sizeof(SExternalMemSource)); if (pSource == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -233,7 +233,7 @@ static int32_t sortComparClearup(SMsortComparParam* cmpParam) { for(int32_t i = 0; i < cmpParam->numOfSources; ++i) { SExternalMemSource* pSource = cmpParam->pSources[i]; blockDataDestroy(pSource->src.pBlock); - tfree(pSource); + taosMemoryFreeClear(pSource); } cmpParam->numOfSources = 0; @@ -570,7 +570,7 @@ static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) { } } - tfree(source); + taosMemoryFreeClear(source); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 2f1caab30b75b27761f358365a2051bc2270d794..7fb9b2ad7e9bbaaffd8f7ddcd37f90a40114ad2d 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -62,7 +62,7 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { } if (pInfo->pBlock == NULL) { - pInfo->pBlock = static_cast(calloc(1, sizeof(SSDataBlock))); + pInfo->pBlock = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); pInfo->pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); @@ -70,8 +70,8 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { colInfo.info.type = TSDB_DATA_TYPE_INT; colInfo.info.bytes = sizeof(int32_t); colInfo.info.colId = 1; - colInfo.pData = static_cast(calloc(pInfo->numOfRowsPerPage, sizeof(int32_t))); - colInfo.nullbitmap = static_cast(calloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); + colInfo.pData = static_cast(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int32_t))); + colInfo.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo); @@ -82,7 +82,7 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { // // colInfo1.varmeta.allocLen = 0;//numOfRows * sizeof(int32_t); // colInfo1.varmeta.length = 0; -// colInfo1.varmeta.offset = static_cast(calloc(1, numOfRows * sizeof(int32_t))); +// colInfo1.varmeta.offset = static_cast(taosMemoryCalloc(1, numOfRows * sizeof(int32_t))); // // taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo1); } else { @@ -128,7 +128,7 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { } if (pInfo->pBlock == NULL) { - pInfo->pBlock = static_cast(calloc(1, sizeof(SSDataBlock))); + pInfo->pBlock = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); pInfo->pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); @@ -136,8 +136,8 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { colInfo.info.type = TSDB_DATA_TYPE_TIMESTAMP; colInfo.info.bytes = sizeof(int64_t); colInfo.info.colId = 1; - colInfo.pData = static_cast(calloc(pInfo->numOfRowsPerPage, sizeof(int64_t))); -// colInfo.nullbitmap = static_cast(calloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); + colInfo.pData = static_cast(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int64_t))); +// colInfo.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo); @@ -146,8 +146,8 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { colInfo1.info.bytes = 4; colInfo1.info.colId = 2; - colInfo1.pData = static_cast(calloc(pInfo->numOfRowsPerPage, sizeof(int32_t))); - colInfo1.nullbitmap = static_cast(calloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); + colInfo1.pData = static_cast(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int32_t))); + colInfo1.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo1); } else { @@ -195,7 +195,7 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { } SOperatorInfo* createDummyOperator(int32_t startVal, int32_t numOfBlocks, int32_t rowsPerPage, int32_t type, int32_t numOfCols) { - SOperatorInfo* pOperator = static_cast(calloc(1, sizeof(SOperatorInfo))); + SOperatorInfo* pOperator = static_cast(taosMemoryCalloc(1, sizeof(SOperatorInfo))); pOperator->name = "dummyInputOpertor4Test"; if (numOfCols == 1) { @@ -204,7 +204,7 @@ SOperatorInfo* createDummyOperator(int32_t startVal, int32_t numOfBlocks, int32_ pOperator->getNextFn = get2ColsDummyBlock; } - SDummyInputInfo *pInfo = (SDummyInputInfo*) calloc(1, sizeof(SDummyInputInfo)); + SDummyInputInfo *pInfo = (SDummyInputInfo*) taosMemoryCalloc(1, sizeof(SDummyInputInfo)); pInfo->totalPages = numOfBlocks; pInfo->startVal = startVal; pInfo->numOfRowsPerPage = rowsPerPage; @@ -958,11 +958,11 @@ TEST(testCase, inMem_sort_Test) { taosArrayPush(pOrderVal, &o); SArray* pExprInfo = taosArrayInit(4, sizeof(SExprInfo)); - SExprInfo *exp = static_cast(calloc(1, sizeof(SExprInfo))); + SExprInfo *exp = static_cast(taosMemoryCalloc(1, sizeof(SExprInfo))); exp->base.resSchema = createSchema(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1, "res"); taosArrayPush(pExprInfo, &exp); - SExprInfo *exp1 = static_cast(calloc(1, sizeof(SExprInfo))); + SExprInfo *exp1 = static_cast(taosMemoryCalloc(1, sizeof(SExprInfo))); exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1"); taosArrayPush(pExprInfo, &exp1); @@ -1002,10 +1002,10 @@ int32_t cmp(const void* p1, const void* p2) { #if 0 TEST(testCase, external_sort_Test) { #if 0 - su* v = static_cast(calloc(1000000, sizeof(su))); + su* v = static_cast(taosMemoryCalloc(1000000, sizeof(su))); for(int32_t i = 0; i < 1000000; ++i) { v[i].v = taosRand(); - v[i].c = static_cast(malloc(4)); + v[i].c = static_cast(taosMemoryMalloc(4)); *(int32_t*) v[i].c = i; } @@ -1027,11 +1027,11 @@ TEST(testCase, external_sort_Test) { taosArrayPush(pOrderVal, &o); SArray* pExprInfo = taosArrayInit(4, sizeof(SExprInfo)); - SExprInfo *exp = static_cast(calloc(1, sizeof(SExprInfo))); + SExprInfo *exp = static_cast(taosMemoryCalloc(1, sizeof(SExprInfo))); exp->base.resSchema = createSchema(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1, "res"); taosArrayPush(pExprInfo, &exp); - SExprInfo *exp1 = static_cast(calloc(1, sizeof(SExprInfo))); + SExprInfo *exp1 = static_cast(taosMemoryCalloc(1, sizeof(SExprInfo))); exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1"); // taosArrayPush(pExprInfo, &exp1); @@ -1071,8 +1071,8 @@ TEST(testCase, external_sort_Test) { printf("total:%ld\n", s2 - s1); pOperator->closeFn(pOperator->info, 2); - tfree(exp); - tfree(exp1); + taosMemoryFreeClear(exp); + taosMemoryFreeClear(exp1); taosArrayDestroy(pExprInfo); taosArrayDestroy(pOrderVal); } @@ -1088,21 +1088,21 @@ TEST(testCase, sorted_merge_Test) { taosArrayPush(pOrderVal, &o); SArray* pExprInfo = taosArrayInit(4, sizeof(SExprInfo)); - SExprInfo *exp = static_cast(calloc(1, sizeof(SExprInfo))); + SExprInfo *exp = static_cast(taosMemoryCalloc(1, sizeof(SExprInfo))); exp->base.resSchema = createSchema(TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 1, "count_result"); - exp->base.pColumns = static_cast(calloc(1, sizeof(SColumn))); + exp->base.pColumns = static_cast(taosMemoryCalloc(1, sizeof(SColumn))); exp->base.pColumns->flag = TSDB_COL_NORMAL; exp->base.pColumns->info = (SColumnInfo) {.colId = 1, .type = TSDB_DATA_TYPE_INT, .bytes = 4}; exp->base.numOfCols = 1; taosArrayPush(pExprInfo, &exp); - SExprInfo *exp1 = static_cast(calloc(1, sizeof(SExprInfo))); + SExprInfo *exp1 = static_cast(taosMemoryCalloc(1, sizeof(SExprInfo))); exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1"); // taosArrayPush(pExprInfo, &exp1); int32_t numOfSources = 10; - SOperatorInfo** plist = (SOperatorInfo**) calloc(numOfSources, sizeof(void*)); + SOperatorInfo** plist = (SOperatorInfo**) taosMemoryCalloc(numOfSources, sizeof(void*)); for(int32_t i = 0; i < numOfSources; ++i) { plist[i] = createDummyOperator(1, 1, 1, data_asc, 1); } @@ -1143,8 +1143,8 @@ TEST(testCase, sorted_merge_Test) { printf("total:%ld\n", s2 - s1); pOperator->closeFn(pOperator->info, 2); - tfree(exp); - tfree(exp1); + taosMemoryFreeClear(exp); + taosMemoryFreeClear(exp1); taosArrayDestroy(pExprInfo); taosArrayDestroy(pOrderVal); } @@ -1160,18 +1160,18 @@ TEST(testCase, time_interval_Operator_Test) { taosArrayPush(pOrderVal, &o); SArray* pExprInfo = taosArrayInit(4, sizeof(SExprInfo)); - SExprInfo *exp = static_cast(calloc(1, sizeof(SExprInfo))); + SExprInfo *exp = static_cast(taosMemoryCalloc(1, sizeof(SExprInfo))); exp->base.resSchema = createSchema(TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1, "ts"); - exp->base.pColumns = static_cast(calloc(1, sizeof(SColumn))); + exp->base.pColumns = static_cast(taosMemoryCalloc(1, sizeof(SColumn))); exp->base.pColumns->flag = TSDB_COL_NORMAL; exp->base.pColumns->info = (SColumnInfo) {.colId = 1, .type = TSDB_DATA_TYPE_TIMESTAMP, .bytes = 8}; exp->base.numOfCols = 1; taosArrayPush(pExprInfo, &exp); - SExprInfo *exp1 = static_cast(calloc(1, sizeof(SExprInfo))); + SExprInfo *exp1 = static_cast(taosMemoryCalloc(1, sizeof(SExprInfo))); exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BIGINT, 8, 2, "res1"); - exp1->base.pColumns = static_cast(calloc(1, sizeof(SColumn))); + exp1->base.pColumns = static_cast(taosMemoryCalloc(1, sizeof(SColumn))); exp1->base.pColumns->flag = TSDB_COL_NORMAL; exp1->base.pColumns->info = (SColumnInfo) {.colId = 1, .type = TSDB_DATA_TYPE_INT, .bytes = 4}; exp1->base.numOfCols = 1; @@ -1221,8 +1221,8 @@ TEST(testCase, time_interval_Operator_Test) { printf("total:%ld\n", s2 - s1); pOperator->closeFn(pOperator->info, 2); - tfree(exp); - tfree(exp1); + taosMemoryFreeClear(exp); + taosMemoryFreeClear(exp1); taosArrayDestroy(pExprInfo); taosArrayDestroy(pOrderVal); } diff --git a/source/libs/executor/test/sortTests.cpp b/source/libs/executor/test/sortTests.cpp index 19567943954b07874eae5f5807b1d7066a1cfb30..586aed7a67c7705993f6d16bd68cad6662fa1d99 100644 --- a/source/libs/executor/test/sortTests.cpp +++ b/source/libs/executor/test/sortTests.cpp @@ -47,15 +47,15 @@ SSDataBlock* getSingleColDummyBlock(void* param) { return NULL; } - SSDataBlock* pBlock = static_cast(calloc(1, sizeof(SSDataBlock))); + SSDataBlock* pBlock = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); SColumnInfoData colInfo = {0}; colInfo.info.type = TSDB_DATA_TYPE_INT; colInfo.info.bytes = sizeof(int32_t); colInfo.info.colId = 1; - colInfo.pData = static_cast(calloc(pInfo->pageRows, sizeof(int32_t))); - colInfo.nullbitmap = static_cast(calloc(1, (pInfo->pageRows + 7) / 8)); + colInfo.pData = static_cast(taosMemoryCalloc(pInfo->pageRows, sizeof(int32_t))); + colInfo.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->pageRows + 7) / 8)); taosArrayPush(pBlock->pDataBlock, &colInfo); @@ -203,12 +203,12 @@ TEST(testCase, external_mem_sort_Test) { SSortHandle* phandle = tsortCreateSortHandle(orderInfo, false, SORT_SINGLESOURCE_SORT, 1024, 5, &s, 1, "test_abc"); tsortSetFetchRawDataFp(phandle, getSingleColDummyBlock); - _info* pInfo = (_info*) calloc(1, sizeof(_info)); + _info* pInfo = (_info*) taosMemoryCalloc(1, sizeof(_info)); pInfo->startVal = 100000; pInfo->pageRows = 1000; pInfo->count = 50; - SGenericSource* ps = static_cast(calloc(1, sizeof(SGenericSource))); + SGenericSource* ps = static_cast(taosMemoryCalloc(1, sizeof(SGenericSource))); ps->param = pInfo; tsortAddSource(phandle, ps); @@ -249,8 +249,8 @@ TEST(testCase, ordered_merge_sort_Test) { tsortSetComparFp(phandle, docomp); for(int32_t i = 0; i < 10; ++i) { - SGenericSource* p = static_cast(calloc(1, sizeof(SGenericSource))); - _info* c = static_cast<_info*>(calloc(1, sizeof(_info))); + SGenericSource* p = static_cast(taosMemoryCalloc(1, sizeof(SGenericSource))); + _info* c = static_cast<_info*>(taosMemoryCalloc(1, sizeof(_info))); c->count = 1; c->pageRows = 1000; c->startVal = 0; diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt index 5d4bf2726e8e93fdfe95857a1b8a2d340a2705ff..e1c7626f014b07b49f20d7b6150048793798da05 100644 --- a/source/libs/function/CMakeLists.txt +++ b/source/libs/function/CMakeLists.txt @@ -1,12 +1,57 @@ aux_source_directory(src FUNCTION_SRC) +list(REMOVE_ITEM FUNCTION_SRC src/udfd.c) add_library(function STATIC ${FUNCTION_SRC}) target_include_directories( function - PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/function" + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/function" + "${CMAKE_SOURCE_DIR}/contrib/libuv/include" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( function PRIVATE os util common nodes scalar + PUBLIC uv_a ) + +add_executable(runUdf test/runUdf.c) +target_include_directories( + runUdf + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/function" + "${CMAKE_SOURCE_DIR}/contrib/libuv/include" + "${CMAKE_SOURCE_DIR}/include/os" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) +target_link_libraries( + runUdf + PUBLIC uv_a + PRIVATE os util common nodes function +) + +add_library(udf1 MODULE test/udf1.c) +target_include_directories( + udf1 + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/function" + "${CMAKE_SOURCE_DIR}/include/os" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +#SET(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/build/bin) +add_executable(udfd src/udfd.c) +target_include_directories( + udfd + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/function" + "${CMAKE_SOURCE_DIR}/contrib/libuv/include" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries( + udfd + PUBLIC uv_a + PRIVATE os util common nodes function +) + diff --git a/source/libs/function/inc/tfill.h b/source/libs/function/inc/tfill.h index 81348fba1dd6529b7be2b8c3ff88dbf57323a6a8..b90dbf7799a904609a293861d9de68775b2ca100 100644 --- a/source/libs/function/inc/tfill.h +++ b/source/libs/function/inc/tfill.h @@ -61,7 +61,7 @@ typedef struct SFillInfo { SFillColInfo* pFillCol; // column info for fill operations SFillTagColInfo* pTags; // tags value for filling gap - void* handle; // for debug purpose + const char* id; } SFillInfo; int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows); diff --git a/source/libs/function/inc/tudf.h b/source/libs/function/inc/tudf.h index 163fbdf4bb204be1c00022505ea13cf32854ff99..72875239d2c7f335d08ccf1c3af73108e4b04ef9 100644 --- a/source/libs/function/inc/tudf.h +++ b/source/libs/function/inc/tudf.h @@ -20,68 +20,116 @@ extern "C" { #endif -#include "os.h" -#include "taoserror.h" +//====================================================================================== +//begin API to taosd and qworker +/** + * start udf dameon service + * @return error code + */ +int32_t startUdfService(); + +/** + * stop udf dameon service + * @return error code + */ +int32_t stopUdfService(); enum { - TSDB_UDF_FUNC_NORMAL = 0, - TSDB_UDF_FUNC_INIT, - TSDB_UDF_FUNC_FINALIZE, - TSDB_UDF_FUNC_MERGE, - TSDB_UDF_FUNC_DESTROY, - TSDB_UDF_FUNC_MAX_NUM + TSDB_UDF_TYPE_SCALAR = 0, + TSDB_UDF_TYPE_AGGREGATE = 1 }; -typedef struct SUdfInit { - int32_t maybe_null; /* 1 if function can return NULL */ - uint32_t decimals; /* for real functions */ - uint64_t length; /* For string functions */ - char* ptr; /* free pointer for function data */ - int32_t const_item; /* 0 if result is independent of arguments */ - - // script like lua/javascript - void* script_ctx; - void (*destroyCtxFunc)(void* script_ctx); -} SUdfInit; +enum { + TSDB_UDF_SCRIPT_BIN_LIB = 0, + TSDB_UDF_SCRIPT_LUA = 1, +}; typedef struct SUdfInfo { - int32_t functionId; // system assigned function id - int32_t funcType; // scalar function or aggregate function - int8_t resType; // result type - int16_t resBytes; // result byte - int32_t contLen; // content length - int32_t bufSize; // interbuf size - char* name; // function name - void* handle; // handle loaded in mem - void* funcs[TSDB_UDF_FUNC_MAX_NUM]; // function ptr - - // for script like lua/javascript only - int isScript; - void* pScriptCtx; - - SUdfInit init; - char* content; - char* path; + char *udfName; // function name + int32_t udfType; // scalar function or aggregate function + int8_t scriptType; + char *path; + + int8_t resType; // result type + int16_t resBytes; // result byte + int32_t bufSize; //interbuf size + } SUdfInfo; +typedef void *UdfHandle; + +/** + * setup udf + * @param udf, in + * @param handle, out + * @return error code + */ +int32_t setupUdf(SUdfInfo* udf, UdfHandle *handle); + + +enum { + TSDB_UDF_STEP_NORMAL = 0, + TSDB_UDF_STEP_MERGE, + TSDb_UDF_STEP_FINALIZE, + TSDB_UDF_STEP_MAX_NUM +}; +/** + * call udf + * @param handle udf handle + * @param step + * @param state + * @param stateSize + * @param input + * @param newstate + * @param newStateSize + * @param output + * @return error code + */ + +//TODO: must change the following after metadata flow and data flow between qworker and udfd is well defined +typedef struct SUdfDataBlock { + char* data; + int32_t size; +} SUdfDataBlock; + +int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, SUdfDataBlock input, char **newstate, + int32_t *newStateSize, SUdfDataBlock *output); + +/** + * tearn down udf + * @param handle + * @return + */ +int32_t teardownUdf(UdfHandle handle); + +// end API to taosd and qworker +//============================================================================================================================= +// TODO: Must change +// begin API to UDF writer. + // script -typedef int32_t (*scriptInitFunc)(void* pCtx); -typedef void (*scriptNormalFunc)(void* pCtx, char* data, int16_t iType, int16_t iBytes, int32_t numOfRows, - int64_t* ptList, int64_t key, char* dataOutput, char* tsOutput, int32_t* numOfOutput, - int16_t oType, int16_t oBytes); -typedef void (*scriptFinalizeFunc)(void* pCtx, int64_t key, char* dataOutput, int32_t* numOfOutput); -typedef void (*scriptMergeFunc)(void* pCtx, char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput); -typedef void (*scriptDestroyFunc)(void* pCtx); +//typedef int32_t (*scriptInitFunc)(void* pCtx); +//typedef void (*scriptNormalFunc)(void* pCtx, char* data, int16_t iType, int16_t iBytes, int32_t numOfRows, +// int64_t* ptList, int64_t key, char* dataOutput, char* tsOutput, int32_t* numOfOutput, +// int16_t oType, int16_t oBytes); +//typedef void (*scriptFinalizeFunc)(void* pCtx, int64_t key, char* dataOutput, int32_t* numOfOutput); +//typedef void (*scriptMergeFunc)(void* pCtx, char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput); +//typedef void (*scriptDestroyFunc)(void* pCtx); // dynamic lib -typedef void (*udfNormalFunc)(char* data, int16_t itype, int16_t iBytes, int32_t numOfRows, int64_t* ts, - char* dataOutput, char* interBuf, char* tsOutput, int32_t* numOfOutput, int16_t oType, - int16_t oBytes, SUdfInit* buf); -typedef int32_t (*udfInitFunc)(SUdfInit* data); -typedef void (*udfFinalizeFunc)(char* dataOutput, char* interBuf, int32_t* numOfOutput, SUdfInit* buf); -typedef void (*udfMergeFunc)(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf); -typedef void (*udfDestroyFunc)(SUdfInit* buf); +typedef int32_t (*TUdfInitFunc)(); +typedef void (*TUdfDestroyFunc)(); + +typedef void (*TUdfFunc)(int8_t step, + char *state, int32_t stateSize, SUdfDataBlock input, + char **newstate, int32_t *newStateSize, SUdfDataBlock *output); + +//typedef void (*udfMergeFunc)(char *data, int32_t numOfRows, char *dataOutput, int32_t* numOfOutput); +//typedef void (*udfFinalizeFunc)(char* state, int32_t stateSize, SUdfDataBlock *output); + +// end API to UDF writer +//======================================================================================================================= #ifdef __cplusplus } diff --git a/source/libs/function/inc/tudfInt.h b/source/libs/function/inc/tudfInt.h new file mode 100644 index 0000000000000000000000000000000000000000..5f757c1ef0a36d47d52a1b74670d3f1b888b676f --- /dev/null +++ b/source/libs/function/inc/tudfInt.h @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TUDF_INT_H +#define TDENGINE_TUDF_INT_H + +#ifdef __cplusplus +extern "C" { +#endif + +//TODO replaces them with fnDebug +//#define debugPrint(...) taosPrintLog("Function", DEBUG_INFO, 135, __VA_ARGS__) +#define debugPrint(...) {fprintf(stderr, __VA_ARGS__);fprintf(stderr, "\n");} +enum { + UDF_TASK_SETUP = 0, + UDF_TASK_CALL = 1, + UDF_TASK_TEARDOWN = 2 + +}; + +typedef struct SUdfSetupRequest { + char udfName[16]; // + int8_t scriptType; // 0:c, 1: lua, 2:js + int8_t udfType; //udaf, udf + int16_t pathSize; + char *path; +} SUdfSetupRequest; + +typedef struct SUdfSetupResponse { + int64_t udfHandle; +} SUdfSetupResponse; + + +typedef struct SUdfCallRequest { + int64_t udfHandle; + int8_t step; + + int32_t inputBytes; + char *input; + + int32_t stateBytes; + char *state; +} SUdfCallRequest; + + +typedef struct SUdfCallResponse { + int32_t outputBytes; + char *output; + int32_t newStateBytes; + char *newState; +} SUdfCallResponse; + + +typedef struct SUdfTeardownRequest { + int64_t udfHandle; +} SUdfTeardownRequest; + + +typedef struct SUdfTeardownResponse { +} SUdfTeardownResponse; + +typedef struct SUdfRequest { + int32_t msgLen; + int64_t seqNum; + + int8_t type; + void *subReq; +} SUdfRequest; + +typedef struct SUdfResponse { + int32_t msgLen; + int64_t seqNum; + + int8_t type; + int32_t code; + void *subRsp; +} SUdfResponse; + +int32_t decodeRequest(char *buf, int32_t bufLen, SUdfRequest **pRequest); +int32_t encodeResponse(char **buf, int32_t *bufLen, SUdfResponse *response); +int32_t encodeRequest(char **buf, int32_t *bufLen, SUdfRequest *request); +int32_t decodeResponse(char *buf, int32_t bufLen, SUdfResponse **pResponse); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TUDF_INT_H diff --git a/source/libs/function/inc/udfc.h b/source/libs/function/inc/udfc.h new file mode 100644 index 0000000000000000000000000000000000000000..4d2aeb7049c21a07d1875c2ad9169ee3b5c9212b --- /dev/null +++ b/source/libs/function/inc/udfc.h @@ -0,0 +1,110 @@ +// +// Created by shenglian on 28/02/22. +// + +#ifndef UDF_UDF_H +#define UDF_UDF_H + +#include +#define DEBUG +#ifdef DEBUG +#define debugPrint(...) fprintf(__VA_ARGS__) +#else +#define debugPrint(...) /**/ +#endif + +enum { + UDF_TASK_SETUP = 0, + UDF_TASK_CALL = 1, + UDF_TASK_TEARDOWN = 2 + +}; + +typedef struct SSDataBlock{ + char *data; + int32_t size; +} SSDataBlock; + +typedef struct SUdfInfo { + char *udfName; + char *path; +} SUdfInfo; + +typedef void *UdfHandle; + +int32_t startUdfService(); + +int32_t stopUdfService(); + +//int32_t setupUdf(SUdfInfo *udf, int32_t numOfUdfs, UdfHandle *handles); + +int32_t setupUdf(SUdfInfo* udf, UdfHandle* handle); + +int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, SSDataBlock input, char **newstate, + int32_t *newStateSize, SSDataBlock *output); + +int32_t teardownUdf(UdfHandle handle); + +typedef struct SUdfSetupRequest { + char udfName[16]; // + int8_t scriptType; // 0:c, 1: lua, 2:js + int8_t udfType; //udaf, udf, udtf + int16_t pathSize; + char *path; +} SUdfSetupRequest; + +typedef struct SUdfSetupResponse { + int64_t udfHandle; +} SUdfSetupResponse; + + +typedef struct SUdfCallRequest { + int64_t udfHandle; + int8_t step; + + int32_t inputBytes; + char *input; + + int32_t stateBytes; + char *state; +} SUdfCallRequest; + + +typedef struct SUdfCallResponse { + int32_t outputBytes; + char *output; + int32_t newStateBytes; + char *newState; +} SUdfCallResponse; + + +typedef struct SUdfTeardownRequest { + int64_t udfHandle; +} SUdfTeardownRequest; + + +typedef struct SUdfTeardownResponse { +} SUdfTeardownResponse; + +typedef struct SUdfRequest { + int32_t msgLen; + int64_t seqNum; + + int8_t type; + void *subReq; +} SUdfRequest; + +typedef struct SUdfResponse { + int32_t msgLen; + int64_t seqNum; + + int8_t type; + int32_t code; + void *subRsp; +} SUdfResponse; + +int32_t decodeRequest(char *buf, int32_t bufLen, SUdfRequest **pRequest); +int32_t encodeResponse(char **buf, int32_t *bufLen, SUdfResponse *response); +int32_t encodeRequest(char **buf, int32_t *bufLen, SUdfRequest *request); +int32_t decodeResponse(char *buf, int32_t bufLen, SUdfResponse **pResponse); +#endif //UDF_UDF_H diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 98170208178d58244a8d662690689b431d1be916..610e5a0bb231a3f1693f4d18869fa53972ecfcdd 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -528,11 +528,10 @@ void firstFunction(SqlFunctionCtx *pCtx) { char* buf = GET_ROWCELL_INTERBUF(pResInfo); SInputColumnInfoData* pInput = &pCtx->input; - SColumnInfoData* pInputCol = pInput->pData[0]; // All null data column, return directly. - if (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows) { + if (pInput->colDataAggIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { ASSERT(pInputCol->hasNull == true); return; } diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index 4360515328d286304688a9949040f3e2df86335b..05ed30c61a231176739eb962ad53d1c5b17aa94d 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -1929,7 +1929,7 @@ static void copyTopBotRes(SqlFunctionCtx *pCtx, int32_t type) { // set the corresponding tag data for each record // todo check malloc failure -// char **pData = calloc(pCtx->tagInfo.numOfTagCols, POINTER_BYTES); +// char **pData = taosMemoryCalloc(pCtx->tagInfo.numOfTagCols, POINTER_BYTES); // for (int32_t i = 0; i < pCtx->tagInfo.numOfTagCols; ++i) { // pData[i] = pCtx->tagInfo.pTagCtxList[i]->pOutput; // } @@ -1943,7 +1943,7 @@ static void copyTopBotRes(SqlFunctionCtx *pCtx, int32_t type) { // } // } -// tfree(pData); +// taosMemoryFreeClear(pData); } /* @@ -2422,7 +2422,7 @@ static void apercentile_finalizer(SqlFunctionCtx *pCtx) { // double *res = tHistogramUniform(pOutput->pHisto, ratio, 1); // // memcpy(pCtx->pOutput, res, sizeof(double)); -// free(res); +// taosMemoryFree(res); // } else { // setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); // return; @@ -2433,7 +2433,7 @@ static void apercentile_finalizer(SqlFunctionCtx *pCtx) { double *res = tHistogramUniform(pOutput->pHisto, ratio, 1); memcpy(pCtx->pOutput, res, sizeof(double)); - free(res); + taosMemoryFree(res); } else { // no need to free setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); return; @@ -4004,7 +4004,7 @@ static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDi char* outputBuf = NULL; if (comp) { - outputBuf = malloc(originalLen); + outputBuf = taosMemoryMalloc(originalLen); size_t actualLen = compLen; const char* compStr = tbufReadBinary(&br, &actualLen); @@ -4018,7 +4018,7 @@ static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDi pDist->dataBlockInfos = taosArrayFromList(outputBuf, (uint32_t)numSteps, sizeof(SFileBlockInfo)); if (comp) { - tfree(outputBuf); + taosMemoryFreeClear(outputBuf); } } diff --git a/source/libs/function/src/texpr.c b/source/libs/function/src/texpr.c index bede8b80fdeb738792eeb3f1685d774287099e3d..814aa48b556f40a9c5013c19b60a218f3159a133 100644 --- a/source/libs/function/src/texpr.c +++ b/source/libs/function/src/texpr.c @@ -52,10 +52,10 @@ void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)) { } else if (pNode->nodeType == TEXPR_VALUE_NODE) { taosVariantDestroy(pNode->pVal); } else if (pNode->nodeType == TEXPR_COL_NODE) { - tfree(pNode->pSchema); + taosMemoryFreeClear(pNode->pSchema); } - free(pNode); + taosMemoryFree(pNode); } static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { @@ -80,12 +80,12 @@ static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { assert((*pExpr)->_node.pRight == NULL); } else if (type == TEXPR_VALUE_NODE) { taosVariantDestroy((*pExpr)->pVal); - free((*pExpr)->pVal); + taosMemoryFree((*pExpr)->pVal); } else if (type == TEXPR_COL_NODE) { - free((*pExpr)->pSchema); + taosMemoryFree((*pExpr)->pSchema); } - free(*pExpr); + taosMemoryFree(*pExpr); *pExpr = NULL; } @@ -154,7 +154,7 @@ void exprTreeToBinary(SBufferWriter* bw, tExprNode* expr) { // TODO: these three functions should be made global static void* exception_calloc(size_t nmemb, size_t size) { - void* p = calloc(nmemb, size); + void* p = taosMemoryCalloc(nmemb, size); if (p == NULL) { THROW(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -162,7 +162,7 @@ static void* exception_calloc(size_t nmemb, size_t size) { } static void* exception_malloc(size_t size) { - void* p = malloc(size); + void* p = taosMemoryMalloc(size); if (p == NULL) { THROW(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -195,7 +195,7 @@ static tExprNode* exprTreeFromBinaryImpl(SBufferReader* br) { pVal->nType = tbufReadUint32(br); if (pVal->nType == TSDB_DATA_TYPE_BINARY) { tbufReadToBuffer(br, &pVal->nLen, sizeof(pVal->nLen)); - pVal->pz = calloc(1, pVal->nLen + 1); + pVal->pz = taosMemoryCalloc(1, pVal->nLen + 1); tbufReadToBuffer(br, pVal->pz, pVal->nLen); } else { pVal->i = tbufReadInt64(br); @@ -377,7 +377,7 @@ void convertFilterSetFromBinary(void **q, const char *buf, int32_t len, uint32_t bufLen = 128; } - char *tmp = calloc(1, bufLen * TSDB_NCHAR_SIZE); + char *tmp = taosMemoryCalloc(1, bufLen * TSDB_NCHAR_SIZE); for (int32_t i = 0; i < sz; i++) { switch (sType) { @@ -440,7 +440,7 @@ void convertFilterSetFromBinary(void **q, const char *buf, int32_t len, uint32_t taosVariantCreateFromBinary(&tmpVar, (char *)pvar, t, sType); if (bufLen < t) { - tmp = realloc(tmp, t * TSDB_NCHAR_SIZE); + tmp = taosMemoryRealloc(tmp, t * TSDB_NCHAR_SIZE); bufLen = (int32_t)t; } @@ -530,7 +530,7 @@ void convertFilterSetFromBinary(void **q, const char *buf, int32_t len, uint32_t err_ret: taosVariantDestroy(&tmpVar); taosHashCleanup(pObj); - tfree(tmp); + taosMemoryFreeClear(tmp); } tExprNode* exprdup(tExprNode* pNode) { @@ -538,7 +538,7 @@ tExprNode* exprdup(tExprNode* pNode) { return NULL; } - tExprNode* pCloned = calloc(1, sizeof(tExprNode)); + tExprNode* pCloned = taosMemoryCalloc(1, sizeof(tExprNode)); if (pNode->nodeType == TEXPR_BINARYEXPR_NODE) { tExprNode* pLeft = exprdup(pNode->_node.pLeft); tExprNode* pRight = exprdup(pNode->_node.pRight); @@ -547,17 +547,17 @@ tExprNode* exprdup(tExprNode* pNode) { pCloned->_node.pRight = pRight; pCloned->_node.optr = pNode->_node.optr; } else if (pNode->nodeType == TEXPR_VALUE_NODE) { - pCloned->pVal = calloc(1, sizeof(SVariant)); + pCloned->pVal = taosMemoryCalloc(1, sizeof(SVariant)); taosVariantAssign(pCloned->pVal, pNode->pVal); } else if (pNode->nodeType == TEXPR_COL_NODE) { - pCloned->pSchema = calloc(1, sizeof(SSchema)); + pCloned->pSchema = taosMemoryCalloc(1, sizeof(SSchema)); *pCloned->pSchema = *pNode->pSchema; } else if (pNode->nodeType == TEXPR_FUNCTION_NODE) { strcpy(pCloned->_function.functionName, pNode->_function.functionName); int32_t num = pNode->_function.num; pCloned->_function.num = num; - pCloned->_function.pChild = calloc(num, POINTER_BYTES); + pCloned->_function.pChild = taosMemoryCalloc(num, POINTER_BYTES); for(int32_t i = 0; i < num; ++i) { pCloned->_function.pChild[i] = exprdup(pNode->_function.pChild[i]); } diff --git a/source/libs/function/src/tfill.c b/source/libs/function/src/tfill.c index 46d82aa6fbbf1854eeb8415fe21340fd5d16dad5..1f7e83286b3c912db211d9e8b588e12dad9742b8 100644 --- a/source/libs/function/src/tfill.c +++ b/source/libs/function/src/tfill.c @@ -149,7 +149,7 @@ static void initBeforeAfterDataBuf(SFillInfo* pFillInfo, char** next) { return; } - *next = calloc(1, pFillInfo->rowSize); + *next = taosMemoryCalloc(1, pFillInfo->rowSize); for (int i = 1; i < pFillInfo->numOfCols; i++) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; setNull(*next + pCol->col.offset, pCol->col.type, pCol->col.bytes); @@ -258,7 +258,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, void** data, int32_t outputR if (pFillInfo->index >= pFillInfo->numOfRows || pFillInfo->numOfCurrent >= outputRows) { /* the raw data block is exhausted, next value does not exists */ if (pFillInfo->index >= pFillInfo->numOfRows) { - tfree(*next); + taosMemoryFreeClear(*next); } pFillInfo->numOfTotal += pFillInfo->numOfCurrent; @@ -314,7 +314,7 @@ static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t pSchema->type = pColInfo->col.type; pSchema->bytes = pColInfo->col.bytes; - pFillInfo->pTags[k].tagVal = calloc(1, pColInfo->col.bytes); + pFillInfo->pTags[k].tagVal = taosMemoryCalloc(1, pColInfo->col.bytes); pColInfo->tagIndex = k; k += 1; @@ -342,12 +342,12 @@ static int32_t taosNumOfRemainRows(SFillInfo* pFillInfo) { struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, - struct SFillColInfo* pCol, void* handle) { + struct SFillColInfo* pCol, const char* id) { if (fillType == TSDB_FILL_NONE) { return NULL; } - SFillInfo* pFillInfo = calloc(1, sizeof(SFillInfo)); + SFillInfo* pFillInfo = taosMemoryCalloc(1, sizeof(SFillInfo)); taosResetFillInfo(pFillInfo, skey); pFillInfo->order = order; @@ -357,17 +357,17 @@ struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTag pFillInfo->numOfCols = numOfCols; pFillInfo->precision = precision; pFillInfo->alloc = capacity; - pFillInfo->handle = handle; + pFillInfo->id = id; pFillInfo->interval.interval = slidingTime; pFillInfo->interval.intervalUnit = slidingUnit; pFillInfo->interval.sliding = slidingTime; pFillInfo->interval.slidingUnit = slidingUnit; - pFillInfo->pData = malloc(POINTER_BYTES * numOfCols); + pFillInfo->pData = taosMemoryMalloc(POINTER_BYTES * numOfCols); // if (numOfTags > 0) { - pFillInfo->pTags = calloc(numOfCols, sizeof(SFillTagColInfo)); + pFillInfo->pTags = taosMemoryCalloc(numOfCols, sizeof(SFillTagColInfo)); for (int32_t i = 0; i < numOfCols; ++i) { pFillInfo->pTags[i].col.colId = -2; // TODO } @@ -394,19 +394,19 @@ void* taosDestroyFillInfo(SFillInfo* pFillInfo) { return NULL; } - tfree(pFillInfo->prevValues); - tfree(pFillInfo->nextValues); + taosMemoryFreeClear(pFillInfo->prevValues); + taosMemoryFreeClear(pFillInfo->nextValues); for(int32_t i = 0; i < pFillInfo->numOfTags; ++i) { - tfree(pFillInfo->pTags[i].tagVal); + taosMemoryFreeClear(pFillInfo->pTags[i].tagVal); } - tfree(pFillInfo->pTags); + taosMemoryFreeClear(pFillInfo->pTags); - tfree(pFillInfo->pData); - tfree(pFillInfo->pFillCol); + taosMemoryFreeClear(pFillInfo->pData); + taosMemoryFreeClear(pFillInfo->pFillCol); - tfree(pFillInfo); + taosMemoryFreeClear(pFillInfo); return NULL; } @@ -530,7 +530,7 @@ int64_t getFillInfoStart(struct SFillInfo *pFillInfo) { struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const int64_t* fillVal) { int32_t offset = 0; - struct SFillColInfo* pFillCol = calloc(numOfOutput, sizeof(SFillColInfo)); + struct SFillColInfo* pFillCol = taosMemoryCalloc(numOfOutput, sizeof(SFillColInfo)); if (pFillCol == NULL) { return NULL; } diff --git a/source/libs/function/src/thistogram.c b/source/libs/function/src/thistogram.c index 49799aef7aa930d39e0e1c3a29aff2b399d6e6c7..2e60498aba4d339edb5346107c44fac109d54761 100644 --- a/source/libs/function/src/thistogram.c +++ b/source/libs/function/src/thistogram.c @@ -34,11 +34,11 @@ static int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double SHistogramInfo* tHistogramCreate(int32_t numOfEntries) { /* need one redundant slot */ - SHistogramInfo* pHisto = malloc(sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfEntries + 1)); + SHistogramInfo* pHisto = taosMemoryMalloc(sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfEntries + 1)); #if !defined(USE_ARRAYLIST) pHisto->pList = SSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); - SInsertSupporter* pss = malloc(sizeof(SInsertSupporter)); + SInsertSupporter* pss = taosMemoryMalloc(sizeof(SInsertSupporter)); pss->numOfEntries = pHisto->maxEntries; pss->pSkipList = pHisto->pList; @@ -96,7 +96,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { } #else tSkipListKey key = tSkipListCreateKey(TSDB_DATA_TYPE_DOUBLE, &val, tDataTypes[TSDB_DATA_TYPE_DOUBLE].nSize); - SHistBin* entry = calloc(1, sizeof(SHistBin)); + SHistBin* entry = taosMemoryCalloc(1, sizeof(SHistBin)); entry->val = val; tSkipListNode* pResNode = SSkipListPut((*pHisto)->pList, entry, &key, 0); @@ -352,7 +352,7 @@ void tHistogramDestroy(SHistogramInfo** pHisto) { return; } - free(*pHisto); + taosMemoryFree(*pHisto); *pHisto = NULL; } @@ -417,7 +417,7 @@ int64_t tHistogramSum(SHistogramInfo* pHisto, double v) { double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num) { #if defined(USE_ARRAYLIST) - double* pVal = malloc(num * sizeof(double)); + double* pVal = taosMemoryMalloc(num * sizeof(double)); for (int32_t i = 0; i < num; ++i) { double numOfElem = (ratio[i] / 100) * pHisto->numOfElems; @@ -463,7 +463,7 @@ double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num) { } } #else - double* pVal = malloc(num * sizeof(double)); + double* pVal = taosMemoryMalloc(num * sizeof(double)); for (int32_t i = 0; i < num; ++i) { double numOfElem = ratio[i] * pHisto->numOfElems; @@ -535,7 +535,7 @@ SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2 return pResHistogram; } - SHistBin* pHistoBins = calloc(1, sizeof(SHistBin) * (pHisto1->numOfEntries + pHisto2->numOfEntries)); + SHistBin* pHistoBins = taosMemoryCalloc(1, sizeof(SHistBin) * (pHisto1->numOfEntries + pHisto2->numOfEntries)); int32_t i = 0, j = 0, k = 0; while (i < pHisto1->numOfEntries && j < pHisto2->numOfEntries) { @@ -573,6 +573,6 @@ SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2 pResHistogram->numOfEntries = k; memcpy(pResHistogram->elems, pHistoBins, sizeof(SHistBin) * k); - free(pHistoBins); + taosMemoryFree(pHistoBins); return pResHistogram; } diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 06c58430a4ea44838d758a3fe56c2f428c8b89ff..dd57024624f09d807996309c34e3990a39fdab46 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -29,7 +29,7 @@ int32_t getGroupId(int32_t numOfSlots, int32_t slotIndex, int32_t times) { } static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx) { - SFilePage *buffer = (SFilePage *)calloc(1, pMemBucket->bytes * pMemBucket->pSlots[slotIdx].info.size + sizeof(SFilePage)); + SFilePage *buffer = (SFilePage *)taosMemoryCalloc(1, pMemBucket->bytes * pMemBucket->pSlots[slotIdx].info.size + sizeof(SFilePage)); int32_t groupId = getGroupId(pMemBucket->numOfSlots, slotIdx, pMemBucket->times); SIDList list = getDataBufPagesIdList(pMemBucket->pBuffer, groupId); @@ -216,7 +216,7 @@ static void resetSlotInfo(tMemBucket* pBucket) { } tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval) { - tMemBucket *pBucket = (tMemBucket *)calloc(1, sizeof(tMemBucket)); + tMemBucket *pBucket = (tMemBucket *)taosMemoryCalloc(1, sizeof(tMemBucket)); if (pBucket == NULL) { return NULL; } @@ -233,7 +233,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, if (setBoundingBox(&pBucket->range, pBucket->type, minval, maxval) != 0) { // qError("MemBucket:%p, invalid value range: %f-%f", pBucket, minval, maxval); - free(pBucket); + taosMemoryFree(pBucket); return NULL; } @@ -243,13 +243,13 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, pBucket->hashFunc = getHashFunc(pBucket->type); if (pBucket->hashFunc == NULL) { // qError("MemBucket:%p, not support data type %d, failed", pBucket, pBucket->type); - free(pBucket); + taosMemoryFree(pBucket); return NULL; } - pBucket->pSlots = (tMemBucketSlot *)calloc(pBucket->numOfSlots, sizeof(tMemBucketSlot)); + pBucket->pSlots = (tMemBucketSlot *)taosMemoryCalloc(pBucket->numOfSlots, sizeof(tMemBucketSlot)); if (pBucket->pSlots == NULL) { - free(pBucket); + taosMemoryFree(pBucket); return NULL; } @@ -271,8 +271,8 @@ void tMemBucketDestroy(tMemBucket *pBucket) { } destroyDiskbasedBuf(pBucket->pBuffer); - tfree(pBucket->pSlots); - tfree(pBucket); + taosMemoryFreeClear(pBucket->pSlots); + taosMemoryFreeClear(pBucket); } void tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataType) { @@ -449,7 +449,7 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) GET_TYPED_DATA(nd, double, pMemBucket->type, nextVal); double val = (1 - fraction) * td + fraction * nd; - tfree(buffer); + taosMemoryFreeClear(buffer); return val; } else { // incur a second round bucket split diff --git a/source/libs/function/src/tscript.c b/source/libs/function/src/tscript.c index 25cf3cfbb16598abde4c911bdb1c27a50fe4303c..93e8ecf5cfbc7aa67ed7bd12e404d850fa3eb4f4 100644 --- a/source/libs/function/src/tscript.c +++ b/source/libs/function/src/tscript.c @@ -195,7 +195,7 @@ void taosLoadScriptDestroy(void *pInit) { } ScriptCtx* createScriptCtx(char *script, int8_t resType, int16_t resBytes) { - ScriptCtx *pCtx = (ScriptCtx *)calloc(1, sizeof(ScriptCtx)); + ScriptCtx *pCtx = (ScriptCtx *)taosMemoryCalloc(1, sizeof(ScriptCtx)); pCtx->state = SCRIPT_STATE_INIT; pCtx->pEnv = getScriptEnvFromPool(); // pCtx->resType = resType; @@ -229,7 +229,7 @@ ScriptCtx* createScriptCtx(char *script, int8_t resType, int16_t resBytes) { void destroyScriptCtx(void *pCtx) { if (pCtx == NULL) return; addScriptEnvToPool(((ScriptCtx *)pCtx)->pEnv); - free(pCtx); + taosMemoryFree(pCtx); } void luaValueToTaosType(lua_State *lua, char *interBuf, int32_t *numOfOutput, int16_t oType, int16_t oBytes) { @@ -332,12 +332,12 @@ void destroyLuaEnv(lua_State *lua) { int32_t scriptEnvPoolInit() { const int size = 10; // configure or not - pool = malloc(sizeof(ScriptEnvPool)); + pool = taosMemoryMalloc(sizeof(ScriptEnvPool)); taosThreadMutexInit(&pool->mutex, NULL); pool->scriptEnvs = tdListNew(sizeof(ScriptEnv *)); for (int i = 0; i < size; i++) { - ScriptEnv *env = malloc(sizeof(ScriptEnv)); + ScriptEnv *env = taosMemoryMalloc(sizeof(ScriptEnv)); env->funcId = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);; env->lua_state = createLuaEnv(); tdListAppend(pool->scriptEnvs, (void *)(&env)); @@ -360,13 +360,13 @@ void scriptEnvPoolCleanup() { } tdListFree(pool->scriptEnvs); taosThreadMutexDestroy(&pool->mutex); - free(pool); + taosMemoryFree(pool); } void destroyScriptEnv(ScriptEnv *pEnv) { destroyLuaEnv(pEnv->lua_state); taosHashCleanup(pEnv->funcId); - free(pEnv); + taosMemoryFree(pEnv); } ScriptEnv* getScriptEnvFromPool() { diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index b65e637a57496486919f0114dd633077f337d8d1..a1030f6c21d5e42552515fd0e6af372e95d1a3f1 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -1,195 +1,925 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "uv.h" +#include "os.h" +#include "tlog.h" #include "tudf.h" +#include "tudfInt.h" -static char* getUdfFuncName(char* funcname, char* name, int type) { - switch (type) { - case TSDB_UDF_FUNC_NORMAL: - strcpy(funcname, name); +//TODO: when startup, set thread poll size. add it to cfg +//TODO: udfd restart when exist or aborts +//TODO: network error processing. +//TODO: add unit test +//TODO: add lua support +void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf); + +enum { + UV_TASK_CONNECT = 0, + UV_TASK_REQ_RSP = 1, + UV_TASK_DISCONNECT = 2 +}; + +typedef struct SUdfUvSession { + int64_t severHandle; + uv_pipe_t *udfSvcPipe; +} SUdfUvSession; + +typedef struct SClientUvTaskNode { + int8_t type; + int errCode; + + uv_pipe_t *pipe; + + int64_t seqNum; + uv_buf_t reqBuf; + + uv_sem_t taskSem; + uv_buf_t rspBuf; + + struct SClientUvTaskNode *prev; + struct SClientUvTaskNode *next; +} SClientUvTaskNode; + +typedef struct SClientUdfTask { + int8_t type; + + SUdfUvSession *session; + + int32_t errCode; + + union { + struct { + SUdfSetupRequest req; + SUdfSetupResponse rsp; + } _setup; + struct { + SUdfCallRequest req; + SUdfCallResponse rsp; + } _call; + struct { + SUdfTeardownRequest req; + SUdfTeardownResponse rsp; + } _teardown; + }; + + +} SClientUdfTask; + +typedef struct SClientConnBuf { + char *buf; + int32_t len; + int32_t cap; + int32_t total; +} SClientConnBuf; + +typedef struct SClientUvConn { + uv_pipe_t *pipe; + SClientUvTaskNode taskQueue; + SClientConnBuf readBuf; +} SClientUvConn; + +uv_process_t gUdfdProcess; + +uv_barrier_t gUdfInitBarrier; + +uv_loop_t gUdfdLoop; +uv_thread_t gUdfLoopThread; +uv_async_t gUdfLoopTaskAync; + +uv_async_t gUdfLoopStopAsync; + +uv_mutex_t gUdfTaskQueueMutex; +int64_t gUdfTaskSeqNum = 0; + +//double circular linked list +typedef SClientUvTaskNode *SClientUvTaskQueue; +SClientUvTaskNode gUdfQueueNode; +SClientUvTaskQueue gUdfTaskQueue = &gUdfQueueNode; + +//add SClientUvTaskNode task that close conn + + + +void udfTaskQueueInit(SClientUvTaskQueue q) { + q->next = q; + q->prev = q; +} + +bool udfTaskQueueIsEmpty(SClientUvTaskQueue q) { + return q == q->next; +} + +void udfTaskQueueInsertTail(SClientUvTaskQueue q, SClientUvTaskNode *e) { + e->next = q; + e->prev = q->prev; + e->prev->next = e; + q->prev = e; +} + +void udfTaskQueueInsertTaskAtHead(SClientUvTaskQueue q, SClientUvTaskNode *e) { + e->next = q->next; + e->prev = q; + q->next->prev = e; + q->next = e; +} + +void udfTaskQueueRemoveTask(SClientUvTaskNode *e) { + e->prev->next = e->next; + e->next->prev = e->prev; +} + +void udfTaskQueueSplit(SClientUvTaskQueue q, SClientUvTaskNode *from, SClientUvTaskQueue n) { + n->prev = q->prev; + n->prev->next = n; + n->next = from; + q->prev = from->prev; + q->prev->next = q; + from->prev = n; +} + +SClientUvTaskNode *udfTaskQueueHeadTask(SClientUvTaskQueue q) { + return q->next; +} + +SClientUvTaskNode *udfTaskQueueTailTask(SClientUvTaskQueue q) { + return q->prev; +} + +SClientUvTaskNode *udfTaskQueueNext(SClientUvTaskNode *e) { + return e->next; +} + +void udfTaskQueueMove(SClientUvTaskQueue q, SClientUvTaskQueue n) { + if (udfTaskQueueIsEmpty(q)) { + udfTaskQueueInit(n); + } else { + SClientUvTaskNode *h = udfTaskQueueHeadTask(q); + udfTaskQueueSplit(q, h, n); + } +} + + +int32_t encodeRequest(char **pBuf, int32_t *pBufLen, SUdfRequest *request) { + debugPrint("%s", "encoding request"); + + int len = sizeof(SUdfRequest) - sizeof(void *); + switch (request->type) { + case UDF_TASK_SETUP: { + SUdfSetupRequest *setup = (SUdfSetupRequest *) (request->subReq); + len += sizeof(SUdfSetupRequest) - 1 * sizeof(char *) + setup->pathSize; break; - case TSDB_UDF_FUNC_INIT: - sprintf(funcname, "%s_init", name); + } + case UDF_TASK_CALL: { + SUdfCallRequest *call = (SUdfCallRequest *) (request->subReq); + len += sizeof(SUdfCallRequest) - 2 * sizeof(char *) + call->inputBytes + call->stateBytes; break; - case TSDB_UDF_FUNC_FINALIZE: - sprintf(funcname, "%s_finalize", name); + } + case UDF_TASK_TEARDOWN: { + SUdfTeardownRequest *teardown = (SUdfTeardownRequest *) (request->subReq); + len += sizeof(SUdfTeardownRequest); break; - case TSDB_UDF_FUNC_MERGE: - sprintf(funcname, "%s_merge", name); + } + default: break; - case TSDB_UDF_FUNC_DESTROY: - sprintf(funcname, "%s_destroy", name); + } + + char *bufBegin = taosMemoryMalloc(len); + char *buf = bufBegin; + + //skip msgLen first + buf += sizeof(int32_t); + + *(int64_t *) buf = request->seqNum; + buf += sizeof(int64_t); + *(int8_t *) buf = request->type; + buf += sizeof(int8_t); + + switch (request->type) { + case UDF_TASK_SETUP: { + SUdfSetupRequest *setup = (SUdfSetupRequest *) (request->subReq); + memcpy(buf, setup->udfName, 16); + buf += 16; + *(int8_t *) buf = setup->scriptType; + buf += sizeof(int8_t); + *(int8_t *) buf = setup->udfType; + buf += sizeof(int8_t); + *(int16_t *) buf = setup->pathSize; + buf += sizeof(int16_t); + memcpy(buf, setup->path, setup->pathSize); + buf += setup->pathSize; + break; + } + + case UDF_TASK_CALL: { + SUdfCallRequest *call = (SUdfCallRequest *) (request->subReq); + *(int64_t *) buf = call->udfHandle; + buf += sizeof(int64_t); + *(int8_t *) buf = call->step; + buf += sizeof(int8_t); + *(int32_t *) buf = call->inputBytes; + buf += sizeof(int32_t); + memcpy(buf, call->input, call->inputBytes); + buf += call->inputBytes; + *(int32_t *) buf = call->stateBytes; + buf += sizeof(int32_t); + memcpy(buf, call->state, call->stateBytes); + buf += call->stateBytes; break; + } + + case UDF_TASK_TEARDOWN: { + SUdfTeardownRequest *teardown = (SUdfTeardownRequest *) (request->subReq); + *(int64_t *) buf = teardown->udfHandle; + buf += sizeof(int64_t); + break; + } default: - assert(0); break; } - return funcname; + request->msgLen = buf - bufBegin; + *(int32_t *) bufBegin = request->msgLen; + *pBuf = bufBegin; + *pBufLen = request->msgLen; + return 0; } -#if 0 -int32_t initUdfInfo(SUdfInfo* pUdfInfo) { - if (pUdfInfo == NULL) { - return TSDB_CODE_SUCCESS; +int32_t decodeRequest(char *bufMsg, int32_t bufLen, SUdfRequest **pRequest) { + debugPrint("%s", "decoding request"); + if (*(int32_t *) bufMsg != bufLen) { + debugPrint("%s", "decoding request error"); + return -1; } - ////qError("script len: %d", pUdfInfo->contLen); - if (isValidScript(pUdfInfo->content, pUdfInfo->contLen)) { - pUdfInfo->isScript = 1; - pUdfInfo->pScriptCtx = createScriptCtx(pUdfInfo->content, pUdfInfo->resType, pUdfInfo->resBytes); - if (pUdfInfo->pScriptCtx == NULL) { - return TSDB_CODE_QRY_SYS_ERROR; + char *buf = bufMsg; + SUdfRequest *request = taosMemoryMalloc(sizeof(SUdfRequest)); + request->subReq = NULL; + request->msgLen = *(int32_t *) (buf); + buf += sizeof(int32_t); + request->seqNum = *(int64_t *) (buf); + buf += sizeof(int64_t); + request->type = *(int8_t *) (buf); + buf += sizeof(int8_t); + + switch (request->type) { + case UDF_TASK_SETUP: { + SUdfSetupRequest *setup = taosMemoryMalloc(sizeof(SUdfSetupRequest)); + + memcpy(setup->udfName, buf, 16); + buf += 16; + setup->scriptType = *(int8_t *) buf; + buf += sizeof(int8_t); + setup->udfType = *(int8_t *) buf; + buf += sizeof(int8_t); + setup->pathSize = *(int16_t *) buf; + buf += sizeof(int16_t); + setup->path = buf; + buf += setup->pathSize; + + request->subReq = setup; + break; } - tfree(pUdfInfo->content); - - pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] = taosLoadScriptInit; - if (pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] == NULL - || (*(scriptInitFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_INIT])(pUdfInfo->pScriptCtx) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_QRY_SYS_ERROR; + case UDF_TASK_CALL: { + SUdfCallRequest *call = taosMemoryMalloc(sizeof(SUdfCallRequest)); + + call->udfHandle = *(int64_t *) buf; + buf += sizeof(int64_t); + call->step = *(int8_t *) buf; + buf += sizeof(int8_t); + call->inputBytes = *(int32_t *) buf; + buf += sizeof(int32_t); + call->input = buf; + buf += call->inputBytes; + call->stateBytes = *(int32_t *) buf; + buf += sizeof(int32_t); + call->state = buf; + buf += call->stateBytes; + + request->subReq = call; + break; } - pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL] = taosLoadScriptNormal; + case UDF_TASK_TEARDOWN: { + SUdfTeardownRequest *teardown = taosMemoryMalloc(sizeof(SUdfTeardownRequest)); + + teardown->udfHandle = *(int64_t *) buf; + buf += sizeof(int64_t); - if (pUdfInfo->funcType == FUNCTION_TYPE_AGG) { - pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE] = taosLoadScriptFinalize; - pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] = taosLoadScriptMerge; + request->subReq = teardown; } - pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY] = taosLoadScriptDestroy; - } else { - char path[PATH_MAX] = {0}; - taosGetTmpfilePath("script", path, tsTempDir); + } + if (buf - bufMsg != bufLen) { + debugPrint("%s", "decode request error"); + taosMemoryFree(request->subReq); + taosMemoryFree(request); + return -1; + } + *pRequest = request; + return 0; +} + +int32_t encodeResponse(char **pBuf, int32_t *pBufLen, SUdfResponse *response) { + debugPrint("%s", "encoding response"); + + int32_t len = sizeof(SUdfResponse) - sizeof(void *); + + switch (response->type) { + case UDF_TASK_SETUP: { + len += sizeof(SUdfSetupResponse); + break; + } + case UDF_TASK_CALL: { + SUdfCallResponse *callResp = (SUdfCallResponse *) (response->subRsp); + len += sizeof(SUdfCallResponse) - 2 * sizeof(char *) + + callResp->outputBytes + callResp->newStateBytes; + break; + } + case UDF_TASK_TEARDOWN: { + len += sizeof(SUdfTeardownResponse); + break; + } + } - FILE* file = fopen(path, "w+"); + char *bufBegin = taosMemoryMalloc(len); + char *buf = bufBegin; - // TODO check for failure of flush to disk - /*size_t t = */ fwrite(pUdfInfo->content, pUdfInfo->contLen, 1, file); - fclose(file); - tfree(pUdfInfo->content); + //skip msgLen + buf += sizeof(int32_t); - pUdfInfo->path = strdup(path); + *(int64_t *) buf = response->seqNum; + buf += sizeof(int64_t); + *(int8_t *) buf = response->type; + buf += sizeof(int8_t); + *(int32_t *) buf = response->code; + buf += sizeof(int32_t); - pUdfInfo->handle = taosLoadDll(path); - if (NULL == pUdfInfo->handle) { - return TSDB_CODE_QRY_SYS_ERROR; + switch (response->type) { + case UDF_TASK_SETUP: { + SUdfSetupResponse *setupResp = (SUdfSetupResponse *) (response->subRsp); + *(int64_t *) buf = setupResp->udfHandle; + buf += sizeof(int64_t); + break; + } + case UDF_TASK_CALL: { + SUdfCallResponse *callResp = (SUdfCallResponse *) (response->subRsp); + *(int32_t *) buf = callResp->outputBytes; + buf += sizeof(int32_t); + memcpy(buf, callResp->output, callResp->outputBytes); + buf += callResp->outputBytes; + + *(int32_t *) buf = callResp->newStateBytes; + buf += sizeof(int32_t); + memcpy(buf, callResp->newState, callResp->newStateBytes); + buf += callResp->newStateBytes; + break; } + case UDF_TASK_TEARDOWN: { + SUdfTeardownResponse *teardownResp = (SUdfTeardownResponse *) (response->subRsp); + break; + } + default: + break; + } + response->msgLen = buf - bufBegin; + *(int32_t *) bufBegin = response->msgLen; + *pBuf = bufBegin; + *pBufLen = response->msgLen; + return 0; +} - char funcname[FUNCTIONS_NAME_MAX_LENGTH + 10] = {0}; - pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_NORMAL)); - if (NULL == pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL]) { - return TSDB_CODE_QRY_SYS_ERROR; +int32_t decodeResponse(char *bufMsg, int32_t bufLen, SUdfResponse **pResponse) { + debugPrint("%s", "decoding response"); + + if (*(int32_t *) bufMsg != bufLen) { + debugPrint("%s", "can not decode response"); + return -1; + } + char *buf = bufMsg; + SUdfResponse *rsp = taosMemoryMalloc(sizeof(SUdfResponse)); + rsp->msgLen = *(int32_t *) buf; + buf += sizeof(int32_t); + rsp->seqNum = *(int64_t *) buf; + buf += sizeof(int64_t); + rsp->type = *(int8_t *) buf; + buf += sizeof(int8_t); + rsp->code = *(int32_t *) buf; + buf += sizeof(int32_t); + + switch (rsp->type) { + case UDF_TASK_SETUP: { + SUdfSetupResponse *setupRsp = (SUdfSetupResponse *) taosMemoryMalloc(sizeof(SUdfSetupResponse)); + setupRsp->udfHandle = *(int64_t *) buf; + buf += sizeof(int64_t); + rsp->subRsp = (char *) setupRsp; + break; } + case UDF_TASK_CALL: { + SUdfCallResponse *callRsp = (SUdfCallResponse *) taosMemoryMalloc(sizeof(SUdfCallResponse)); + callRsp->outputBytes = *(int32_t *) buf; + buf += sizeof(int32_t); - pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_INIT)); + callRsp->output = buf; + buf += callRsp->outputBytes; - if (pUdfInfo->funcType == FUNCTION_TYPE_AGG) { - pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_FINALIZE)); - pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_MERGE)); + callRsp->newStateBytes = *(int32_t *) buf; + buf += sizeof(int32_t); + + callRsp->newState = buf; + buf += callRsp->newStateBytes; + + rsp->subRsp = callRsp; + break; } + case UDF_TASK_TEARDOWN: { + SUdfTeardownResponse *teardownRsp = (SUdfTeardownResponse *) taosMemoryMalloc(sizeof(SUdfTeardownResponse)); + rsp->subRsp = teardownRsp; + break; + } + default: + break; + } + if (buf - bufMsg != bufLen) { + debugPrint("%s", "can not decode response"); + taosMemoryFree(rsp->subRsp); + taosMemoryFree(rsp); + return -1; + } + *pResponse = rsp; + return 0; +} + +void onUdfdExit(uv_process_t *req, int64_t exit_status, int term_signal) { + debugPrint("Process exited with status %" PRId64 ", signal %d", exit_status, term_signal); + uv_close((uv_handle_t *) req, NULL); + //TODO: restart the udfd process +} - pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_DESTROY)); +void onUdfcPipeClose(uv_handle_t *handle) { + SClientUvConn *conn = handle->data; + if (!udfTaskQueueIsEmpty(&conn->taskQueue)) { + SClientUvTaskNode *task = udfTaskQueueHeadTask(&conn->taskQueue); + task->errCode = 0; + uv_sem_post(&task->taskSem); + } + + taosMemoryFree(conn->readBuf.buf); + taosMemoryFree(conn); + taosMemoryFree((uv_pipe_t *) handle); + +} + +int32_t udfcGetUvTaskResponseResult(SClientUdfTask *task, SClientUvTaskNode *uvTask) { + debugPrint("%s", "get uv task result"); + if (uvTask->type == UV_TASK_REQ_RSP) { + if (uvTask->rspBuf.base != NULL) { + SUdfResponse *rsp; + decodeResponse(uvTask->rspBuf.base, uvTask->rspBuf.len, &rsp); + task->errCode = rsp->code; + + switch (task->type) { + case UDF_TASK_SETUP: { + //TODO: copy or not + task->_setup.rsp = *(SUdfSetupResponse *) (rsp->subRsp); + break; + } + case UDF_TASK_CALL: { + task->_call.rsp = *(SUdfCallResponse *) (rsp->subRsp); + //TODO: copy or not + break; + } + case UDF_TASK_TEARDOWN: { + task->_teardown.rsp = *(SUdfTeardownResponse *) (rsp->subRsp); + //TODO: copy or not? + break; + } + default: { + break; + } + } + + // TODO: the call buffer is setup and freed by udf invocation + taosMemoryFree(uvTask->rspBuf.base); + taosMemoryFree(rsp->subRsp); + taosMemoryFree(rsp); + } else { + task->errCode = uvTask->errCode; + } + } else if (uvTask->type == UV_TASK_CONNECT) { + task->errCode = uvTask->errCode; + } else if (uvTask->type == UV_TASK_DISCONNECT) { + task->errCode = uvTask->errCode; + } + return 0; +} - if (pUdfInfo->funcs[TSDB_UDF_FUNC_INIT]) { - return (*(udfInitFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_INIT])(&pUdfInfo->init); +void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { + debugPrint("%s", "client allocate buffer to receive from pipe"); + SClientUvConn *conn = handle->data; + SClientConnBuf *connBuf = &conn->readBuf; + + int32_t msgHeadSize = sizeof(int32_t) + sizeof(int64_t); + if (connBuf->cap == 0) { + connBuf->buf = taosMemoryMalloc(msgHeadSize); + if (connBuf->buf) { + connBuf->len = 0; + connBuf->cap = msgHeadSize; + connBuf->total = -1; + + buf->base = connBuf->buf; + buf->len = connBuf->cap; + } else { + //TODO: log error + buf->base = NULL; + buf->len = 0; + } + } else { + connBuf->cap = connBuf->total > connBuf->cap ? connBuf->total : connBuf->cap; + void *resultBuf = taosMemoryRealloc(connBuf->buf, connBuf->cap); + if (resultBuf) { + connBuf->buf = resultBuf; + buf->base = connBuf->buf + connBuf->len; + buf->len = connBuf->cap - connBuf->len; + } else { + //TODO: log error free connBuf->buf + buf->base = NULL; + buf->len = 0; } } - return TSDB_CODE_SUCCESS; + debugPrint("\tconn buf cap - len - total : %d - %d - %d", connBuf->cap, connBuf->len, connBuf->total); + } -void destroyUdfInfo(SUdfInfo* pUdfInfo) { - if (pUdfInfo == NULL) { - return; +bool isUdfcUvMsgComplete(SClientConnBuf *connBuf) { + if (connBuf->total == -1 && connBuf->len >= sizeof(int32_t)) { + connBuf->total = *(int32_t *) (connBuf->buf); } + if (connBuf->len == connBuf->cap && connBuf->total == connBuf->cap) { + return true; + } + return false; +} + +void udfcUvHandleRsp(SClientUvConn *conn) { + SClientConnBuf *connBuf = &conn->readBuf; + int64_t seqNum = *(int64_t *) (connBuf->buf + sizeof(int32_t)); // msglen int32_t then seqnum - if (pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY]) { - if (pUdfInfo->isScript) { - (*(scriptDestroyFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY])(pUdfInfo->pScriptCtx); - tfree(pUdfInfo->content); - }else{ - (*(udfDestroyFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY])(&pUdfInfo->init); + if (udfTaskQueueIsEmpty(&conn->taskQueue)) { + //LOG error + return; + } + bool found = false; + SClientUvTaskNode *taskFound = NULL; + SClientUvTaskNode *task = udfTaskQueueNext(&conn->taskQueue); + while (task != &conn->taskQueue) { + if (task->seqNum == seqNum) { + if (found == false) { + found = true; + taskFound = task; + } else { + //LOG error; + continue; + } } + task = udfTaskQueueNext(task); } - tfree(pUdfInfo->name); + if (taskFound) { + taskFound->rspBuf = uv_buf_init(connBuf->buf, connBuf->len); + udfTaskQueueRemoveTask(taskFound); + uv_sem_post(&taskFound->taskSem); + } else { + //LOG error + } + connBuf->buf = NULL; + connBuf->total = -1; + connBuf->len = 0; + connBuf->cap = 0; +} + +void udfcUvHandleError(SClientUvConn *conn) { + uv_close((uv_handle_t *) conn->pipe, onUdfcPipeClose); +} + +void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { + debugPrint("%s, nread: %zd", "client read from pipe", nread); + if (nread == 0) return; + + SClientUvConn *conn = client->data; + SClientConnBuf *connBuf = &conn->readBuf; + if (nread > 0) { + connBuf->len += nread; + if (isUdfcUvMsgComplete(connBuf)) { + udfcUvHandleRsp(conn); + } - if (pUdfInfo->path) { - unlink(pUdfInfo->path); + } + if (nread < 0) { + debugPrint("\tclient read error: %s", uv_strerror(nread)); + if (nread == UV_EOF) { + //TODO: + } + udfcUvHandleError(conn); } - tfree(pUdfInfo->path); - tfree(pUdfInfo->content); - taosCloseDll(pUdfInfo->handle); - tfree(pUdfInfo); } -void doInvokeUdf(struct SUdfInfo* pUdfInfo, SqlFunctionCtx *pCtx, int32_t idx, int32_t type) { - int32_t output = 0; +void onUdfClientWrite(uv_write_t *write, int status) { + debugPrint("%s", "after writing to pipe"); + SClientUvTaskNode *uvTask = write->data; + if (status == 0) { + uv_pipe_t *pipe = uvTask->pipe; + SClientUvConn *conn = pipe->data; + udfTaskQueueInsertTail(&conn->taskQueue, uvTask); + } else { + //TODO Log error; + } + debugPrint("\tlength:%zu", uvTask->reqBuf.len); + taosMemoryFree(write); + taosMemoryFree(uvTask->reqBuf.base); +} - if (pUdfInfo == NULL || pUdfInfo->funcs[type] == NULL) { - //qError("empty udf function, type:%d", type); - return; +void onUdfClientConnect(uv_connect_t *connect, int status) { + SClientUvTaskNode *uvTask = connect->data; + uvTask->errCode = status; + if (status != 0) { + //TODO: LOG error } + uv_read_start((uv_stream_t *) uvTask->pipe, udfcAllocateBuffer, onUdfcRead); + taosMemoryFree(connect); + uv_sem_post(&uvTask->taskSem); +} -// //qDebug("invoke udf function:%s,%p", pUdfInfo->name, pUdfInfo->funcs[type]); +int32_t createUdfcUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskNode **pUvTask) { + SClientUvTaskNode *uvTask = taosMemoryCalloc(1, sizeof(SClientUvTaskNode)); + uvTask->type = uvTaskType; + + if (uvTaskType == UV_TASK_CONNECT) { + } else if (uvTaskType == UV_TASK_REQ_RSP) { + uvTask->pipe = task->session->udfSvcPipe; + SUdfRequest request; + request.type = task->type; + request.seqNum = gUdfTaskSeqNum++; + + if (task->type == UDF_TASK_SETUP) { + request.subReq = &task->_setup.req; + request.type = UDF_TASK_SETUP; + } else if (task->type == UDF_TASK_CALL) { + request.subReq = &task->_call.req; + request.type = UDF_TASK_CALL; + } else if (task->type == UDF_TASK_TEARDOWN) { + request.subReq = &task->_teardown.req; + request.type = UDF_TASK_TEARDOWN; + } else { + //TODO log and return error + } + char *buf = NULL; + int32_t bufLen = 0; + encodeRequest(&buf, &bufLen, &request); + uvTask->reqBuf = uv_buf_init(buf, bufLen); + uvTask->seqNum = request.seqNum; + } else if (uvTaskType == UV_TASK_DISCONNECT) { + uvTask->pipe = task->session->udfSvcPipe; + } + uv_sem_init(&uvTask->taskSem, 0); - switch (type) { - case TSDB_UDF_FUNC_NORMAL: - if (pUdfInfo->isScript) { - (*(scriptNormalFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL])(pUdfInfo->pScriptCtx, - (char *)pCtx->pInput + idx * pCtx->inputType, pCtx->inputType, pCtx->inputBytes, pCtx->size, pCtx->ptsList, pCtx->startTs, pCtx->pOutput, - (char *)pCtx->ptsOutputBuf, &output, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - } else { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); + *pUvTask = uvTask; + return 0; +} - void *interBuf = (void *)GET_ROWCELL_INTERBUF(pResInfo); +int32_t queueUvUdfTask(SClientUvTaskNode *uvTask) { + debugPrint("%s, %d", "queue uv task", uvTask->type); - (*(udfNormalFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL])((char *)pCtx->pInput + idx * pCtx->inputType, pCtx->inputType, pCtx->inputBytes, pCtx->size, pCtx->ptsList, - pCtx->pOutput, interBuf, (char *)pCtx->ptsOutputBuf, &output, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes, &pUdfInfo->init); - } + uv_mutex_lock(&gUdfTaskQueueMutex); + udfTaskQueueInsertTail(gUdfTaskQueue, uvTask); + uv_mutex_unlock(&gUdfTaskQueueMutex); + uv_async_send(&gUdfLoopTaskAync); - if (pUdfInfo->funcType == TSDB_FUNC_TYPE_AGGREGATE) { - pCtx->resultInfo->numOfRes = output; - } else { - pCtx->resultInfo->numOfRes += output; - } + uv_sem_wait(&uvTask->taskSem); + uv_sem_destroy(&uvTask->taskSem); - if (pCtx->resultInfo->numOfRes > 0) { - pCtx->resultInfo->hasResult = DATA_SET_FLAG; - } + return 0; +} - break; +int32_t startUvUdfTask(SClientUvTaskNode *uvTask) { + debugPrint("%s, type %d", "start uv task ", uvTask->type); + switch (uvTask->type) { + case UV_TASK_CONNECT: { + uv_pipe_t *pipe = taosMemoryMalloc(sizeof(uv_pipe_t)); + uv_pipe_init(&gUdfdLoop, pipe, 0); + uvTask->pipe = pipe; - case TSDB_UDF_FUNC_MERGE: - if (pUdfInfo->isScript) { - (*(scriptMergeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE])(pUdfInfo->pScriptCtx, pCtx->pInput, pCtx->size, pCtx->pOutput, &output); - } else { - (*(udfMergeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE])(pCtx->pInput, pCtx->size, pCtx->pOutput, &output, &pUdfInfo->init); - } + SClientUvConn *conn = taosMemoryMalloc(sizeof(SClientUvConn)); + conn->pipe = pipe; + conn->readBuf.len = 0; + conn->readBuf.cap = 0; + conn->readBuf.buf = 0; + conn->readBuf.total = -1; + udfTaskQueueInit(&conn->taskQueue); - // set the output value exist - pCtx->resultInfo->numOfRes = output; - if (output > 0) { - pCtx->resultInfo->hasResult = DATA_SET_FLAG; - } + pipe->data = conn; + + uv_connect_t *connReq = taosMemoryMalloc(sizeof(uv_connect_t)); + connReq->data = uvTask; + uv_pipe_connect(connReq, pipe, "udf.sock", onUdfClientConnect); + break; + } + case UV_TASK_REQ_RSP: { + uv_pipe_t *pipe = uvTask->pipe; + uv_write_t *write = taosMemoryMalloc(sizeof(uv_write_t)); + write->data = uvTask; + uv_write(write, (uv_stream_t *) pipe, &uvTask->reqBuf, 1, onUdfClientWrite); break; + } + case UV_TASK_DISCONNECT: { + SClientUvConn *conn = uvTask->pipe->data; + udfTaskQueueInsertTail(&conn->taskQueue, uvTask); + uv_close((uv_handle_t *) uvTask->pipe, onUdfcPipeClose); + break; + } + default: { + break; + } + } - case TSDB_UDF_FUNC_FINALIZE: { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - void *interBuf = (void *)GET_ROWCELL_INTERBUF(pResInfo); - if (pUdfInfo->isScript) { - (*(scriptFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pUdfInfo->pScriptCtx, pCtx->startTs, pCtx->pOutput, &output); - } else { - (*(udfFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pCtx->pOutput, interBuf, &output, &pUdfInfo->init); - } - // set the output value exist - pCtx->resultInfo->numOfRes = output; - if (output > 0) { - pCtx->resultInfo->hasResult = DATA_SET_FLAG; - } + return 0; +} - break; - } +void udfClientAsyncCb(uv_async_t *async) { + SClientUvTaskNode node; + SClientUvTaskQueue q = &node; + udfTaskQueueInit(q); + + uv_mutex_lock(&gUdfTaskQueueMutex); + udfTaskQueueMove(gUdfTaskQueue, q); + uv_mutex_unlock(&gUdfTaskQueueMutex); + + while (!udfTaskQueueIsEmpty(q)) { + SClientUvTaskNode *task = udfTaskQueueHeadTask(q); + udfTaskQueueRemoveTask(task); + startUvUdfTask(task); + } + +} + +void udfStopAsyncCb(uv_async_t *async) { + uv_stop(&gUdfdLoop); + uv_loop_close(&gUdfdLoop); +} + +void startUdfd(void *argsThread) { + uv_loop_init(&gUdfdLoop); + + //TODO: path + uv_process_options_t options; + static char path[256] = {0}; + size_t cwdSize; + uv_cwd(path, &cwdSize); + strcat(path, "./udfd"); + char* args[2] = {path, NULL}; + options.args = args; + options.file = path; + options.exit_cb = onUdfdExit; + + int err = uv_spawn(&gUdfdLoop, &gUdfdProcess, &options); + if (err != 0) { + debugPrint("can not spawn udfd. path: %s, error: %s", path, uv_strerror(err)); + } + + uv_async_init(&gUdfdLoop, &gUdfLoopTaskAync, udfClientAsyncCb); + uv_async_init(&gUdfdLoop, &gUdfLoopStopAsync, udfStopAsyncCb); + uv_mutex_init(&gUdfTaskQueueMutex); + udfTaskQueueInit(gUdfTaskQueue); + uv_barrier_wait(&gUdfInitBarrier); + uv_run(&gUdfdLoop, UV_RUN_DEFAULT); +} + +int32_t startUdfService() { + uv_barrier_init(&gUdfInitBarrier, 2); + uv_thread_create(&gUdfLoopThread, startUdfd, 0); + uv_barrier_wait(&gUdfInitBarrier); + return 0; +} + +int32_t stopUdfService() { + uv_barrier_destroy(&gUdfInitBarrier); + uv_process_kill(&gUdfdProcess, SIGINT); + uv_async_send(&gUdfLoopStopAsync); + uv_mutex_destroy(&gUdfTaskQueueMutex); + uv_thread_join(&gUdfLoopThread); + return 0; +} + +int32_t udfcRunUvTask(SClientUdfTask *task, int8_t uvTaskType) { + SClientUvTaskNode *uvTask = NULL; + + createUdfcUvTask(task, uvTaskType, &uvTask); + queueUvUdfTask(uvTask); + udfcGetUvTaskResponseResult(task, uvTask); + if (uvTaskType == UV_TASK_CONNECT) { + task->session->udfSvcPipe = uvTask->pipe; + } + taosMemoryFree(uvTask); + uvTask = NULL; + return task->errCode; +} + +int32_t setupUdf(SUdfInfo *udfInfo, UdfHandle *handle) { + debugPrint("%s", "client setup udf"); + SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask)); + task->errCode = 0; + task->session = taosMemoryMalloc(sizeof(SUdfUvSession)); + task->type = UDF_TASK_SETUP; + + SUdfSetupRequest *req = &task->_setup.req; + memcpy(req->udfName, udfInfo->udfName, 16); + req->path = udfInfo->path; + req->pathSize = strlen(req->path) + 1; + req->udfType = udfInfo->udfType; + req->scriptType = udfInfo->scriptType; + + int32_t errCode = udfcRunUvTask(task, UV_TASK_CONNECT); + if (errCode != 0) { + //TODO: log error + return -1; } + + udfcRunUvTask(task, UV_TASK_REQ_RSP); + + SUdfSetupResponse *rsp = &task->_setup.rsp; + task->session->severHandle = rsp->udfHandle; + *handle = task->session; + int32_t err = task->errCode; + taosMemoryFree(task); + return err; } -#endif \ No newline at end of file +int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, SUdfDataBlock input, char **newState, + int32_t *newStateSize, SUdfDataBlock *output) { + debugPrint("%s", "client call udf"); + + SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask)); + task->errCode = 0; + task->session = (SUdfUvSession *) handle; + task->type = UDF_TASK_CALL; + + SUdfCallRequest *req = &task->_call.req; + + req->state = state; + req->stateBytes = stateSize; + req->inputBytes = input.size; + req->input = input.data; + req->udfHandle = task->session->severHandle; + req->step = step; + + udfcRunUvTask(task, UV_TASK_REQ_RSP); + + SUdfCallResponse *rsp = &task->_call.rsp; + *newState = rsp->newState; + *newStateSize = rsp->newStateBytes; + output->size = rsp->outputBytes; + output->data = rsp->output; + int32_t err = task->errCode; + taosMemoryFree(task); + return err; +} + +int32_t teardownUdf(UdfHandle handle) { + debugPrint("%s", "client teardown udf"); + + SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask)); + task->errCode = 0; + task->session = (SUdfUvSession *) handle; + task->type = UDF_TASK_TEARDOWN; + + SUdfTeardownRequest *req = &task->_teardown.req; + req->udfHandle = task->session->severHandle; + + udfcRunUvTask(task, UV_TASK_REQ_RSP); + + + SUdfTeardownResponse *rsp = &task->_teardown.rsp; + + int32_t err = task->errCode; + + udfcRunUvTask(task, UV_TASK_DISCONNECT); + + taosMemoryFree(task->session); + taosMemoryFree(task); + + return err; +} diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c new file mode 100644 index 0000000000000000000000000000000000000000..b473f060c099c1277e6c92752980e8c2e6a7bed6 --- /dev/null +++ b/source/libs/function/src/udfd.c @@ -0,0 +1,356 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "uv.h" +#include "os.h" +#include "tlog.h" + +#include "tudf.h" +#include "tudfInt.h" + + +static uv_loop_t *loop; + +typedef struct SUdfdUvConn { + uv_stream_t *client; + char *inputBuf; + int32_t inputLen; + int32_t inputCap; + int32_t inputTotal; +} SUdfdUvConn; + +typedef struct SUvUdfWork { + uv_stream_t *client; + uv_buf_t input; + uv_buf_t output; +} SUvUdfWork; + +typedef struct SUdf { + int32_t refCount; + + char name[16]; + int8_t type; + + uv_lib_t lib; + TUdfFunc normalFunc; +} SUdf; + +//TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix +//TODO: add private udf structure. +typedef struct SUdfHandle { + SUdf *udf; +} SUdfHandle; + + +void udfdProcessRequest(uv_work_t *req) { + SUvUdfWork *uvUdf = (SUvUdfWork *) (req->data); + SUdfRequest *request = NULL; + decodeRequest(uvUdf->input.base, uvUdf->input.len, &request); + + switch (request->type) { + case UDF_TASK_SETUP: { + debugPrint("%s", "process setup request"); + SUdf *udf = taosMemoryMalloc(sizeof(SUdf)); + udf->refCount = 0; + SUdfSetupRequest *setup = request->subReq; + strcpy(udf->name, setup->udfName); + int err = uv_dlopen(setup->path, &udf->lib); + if (err != 0) { + debugPrint("can not load library %s. error: %s", setup->path, uv_strerror(err)); + //TODO set error + } + + char normalFuncName[32] = {0}; + strcpy(normalFuncName, setup->udfName); + //TODO error, + //TODO find all functions normal, init, destroy, normal, merge, finalize + uv_dlsym(&udf->lib, normalFuncName, (void **) (&udf->normalFunc)); + + SUdfHandle *handle = taosMemoryMalloc(sizeof(SUdfHandle)); + handle->udf = udf; + udf->refCount++; + //TODO: allocate private structure and call init function and set it to handle + SUdfResponse *rsp = taosMemoryMalloc(sizeof(SUdfResponse)); + rsp->seqNum = request->seqNum; + rsp->type = request->type; + rsp->code = 0; + SUdfSetupResponse *subRsp = taosMemoryMalloc(sizeof(SUdfSetupResponse)); + subRsp->udfHandle = (int64_t) (handle); + rsp->subRsp = subRsp; + char *buf; + int32_t len; + encodeResponse(&buf, &len, rsp); + + uvUdf->output = uv_buf_init(buf, len); + + taosMemoryFree(rsp->subRsp); + taosMemoryFree(rsp); + taosMemoryFree(request->subReq); + taosMemoryFree(request); + taosMemoryFree(uvUdf->input.base); + break; + } + + case UDF_TASK_CALL: { + debugPrint("%s", "process call request"); + SUdfCallRequest *call = request->subReq; + SUdfHandle *handle = (SUdfHandle *) (call->udfHandle); + SUdf *udf = handle->udf; + char *newState; + int32_t newStateSize; + SUdfDataBlock input = {.data = call->input, .size= call->inputBytes}; + SUdfDataBlock output; + //TODO: call different functions according to the step + udf->normalFunc(call->step, call->state, call->stateBytes, input, &newState, &newStateSize, &output); + + SUdfResponse *rsp = taosMemoryMalloc(sizeof(SUdfResponse)); + rsp->seqNum = request->seqNum; + rsp->type = request->type; + rsp->code = 0; + SUdfCallResponse *subRsp = taosMemoryMalloc(sizeof(SUdfCallResponse)); + subRsp->outputBytes = output.size; + subRsp->output = output.data; + subRsp->newStateBytes = newStateSize; + subRsp->newState = newState; + rsp->subRsp = subRsp; + + char *buf; + int32_t len; + encodeResponse(&buf, &len, rsp); + uvUdf->output = uv_buf_init(buf, len); + + taosMemoryFree(rsp->subRsp); + taosMemoryFree(rsp); + taosMemoryFree(newState); + taosMemoryFree(output.data); + taosMemoryFree(request->subReq); + taosMemoryFree(request); + taosMemoryFree(uvUdf->input.base); + break; + } + case UDF_TASK_TEARDOWN: { + debugPrint("%s", "process teardown request"); + + SUdfTeardownRequest *teardown = request->subReq; + SUdfHandle *handle = (SUdfHandle *) (teardown->udfHandle); + SUdf *udf = handle->udf; + udf->refCount--; + if (udf->refCount == 0) { + uv_dlclose(&udf->lib); + } + taosMemoryFree(udf); + //TODO: call destroy and free udf private + taosMemoryFree(handle); + + SUdfResponse *rsp = taosMemoryMalloc(sizeof(SUdfResponse)); + rsp->seqNum = request->seqNum; + rsp->type = request->type; + rsp->code = 0; + SUdfTeardownResponse *subRsp = taosMemoryMalloc(sizeof(SUdfTeardownResponse)); + rsp->subRsp = subRsp; + char *buf; + int32_t len; + encodeResponse(&buf, &len, rsp); + uvUdf->output = uv_buf_init(buf, len); + + taosMemoryFree(rsp->subRsp); + taosMemoryFree(rsp); + taosMemoryFree(request->subReq); + taosMemoryFree(request); + taosMemoryFree(uvUdf->input.base); + break; + } + default: { + break; + } + + } + +} + +void udfdOnWrite(uv_write_t *req, int status) { + debugPrint("%s", "after writing to pipe"); + if (status < 0) { + debugPrint("Write error %s", uv_err_name(status)); + } + SUvUdfWork *work = (SUvUdfWork *) req->data; + debugPrint("\tlength: %zu", work->output.len); + taosMemoryFree(work->output.base); + taosMemoryFree(work); + taosMemoryFree(req); +} + + +void udfdSendResponse(uv_work_t *work, int status) { + debugPrint("%s", "send response"); + SUvUdfWork *udfWork = (SUvUdfWork *) (work->data); + + uv_write_t *write_req = taosMemoryMalloc(sizeof(uv_write_t)); + write_req->data = udfWork; + uv_write(write_req, udfWork->client, &udfWork->output, 1, udfdOnWrite); + + taosMemoryFree(work); +} + +void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { + debugPrint("%s", "allocate buffer for read"); + SUdfdUvConn *ctx = handle->data; + int32_t msgHeadSize = sizeof(int32_t) + sizeof(int64_t); + if (ctx->inputCap == 0) { + ctx->inputBuf = taosMemoryMalloc(msgHeadSize); + if (ctx->inputBuf) { + ctx->inputLen = 0; + ctx->inputCap = msgHeadSize; + ctx->inputTotal = -1; + + buf->base = ctx->inputBuf; + buf->len = ctx->inputCap; + } else { + //TODO: log error + buf->base = NULL; + buf->len = 0; + } + } else { + ctx->inputCap = ctx->inputTotal > ctx->inputCap ? ctx->inputTotal : ctx->inputCap; + void *inputBuf = taosMemoryRealloc(ctx->inputBuf, ctx->inputCap); + if (inputBuf) { + ctx->inputBuf = inputBuf; + buf->base = ctx->inputBuf + ctx->inputLen; + buf->len = ctx->inputCap - ctx->inputLen; + } else { + //TODO: log error + buf->base = NULL; + buf->len = 0; + } + } + debugPrint("\tinput buf cap - len - total : %d - %d - %d", ctx->inputCap, ctx->inputLen, ctx->inputTotal); + +} + +bool isUdfdUvMsgComplete(SUdfdUvConn *pipe) { + if (pipe->inputTotal == -1 && pipe->inputLen >= sizeof(int32_t)) { + pipe->inputTotal = *(int32_t *) (pipe->inputBuf); + } + if (pipe->inputLen == pipe->inputCap && pipe->inputTotal == pipe->inputCap) { + return true; + } + return false; +} + +void udfdHandleRequest(SUdfdUvConn *conn) { + uv_work_t *work = taosMemoryMalloc(sizeof(uv_work_t)); + SUvUdfWork *udfWork = taosMemoryMalloc(sizeof(SUvUdfWork)); + udfWork->client = conn->client; + udfWork->input = uv_buf_init(conn->inputBuf, conn->inputLen); + conn->inputBuf = NULL; + conn->inputLen = 0; + conn->inputCap = 0; + conn->inputTotal = -1; + work->data = udfWork; + uv_queue_work(loop, work, udfdProcessRequest, udfdSendResponse); +} + +void udfdPipeCloseCb(uv_handle_t *pipe) { + SUdfdUvConn *conn = pipe->data; + taosMemoryFree(conn->client); + taosMemoryFree(conn->inputBuf); + taosMemoryFree(conn); +} + +void udfdUvHandleError(SUdfdUvConn *conn) { + uv_close((uv_handle_t *) conn->client, udfdPipeCloseCb); +} + +void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { + debugPrint("%s, nread: %zd", "read from pipe", nread); + + if (nread == 0) return; + + SUdfdUvConn *conn = client->data; + + if (nread > 0) { + conn->inputLen += nread; + if (isUdfdUvMsgComplete(conn)) { + udfdHandleRequest(conn); + } else { + //log error or continue; + } + return; + } + + if (nread < 0) { + debugPrint("Read error %s", uv_err_name(nread)); + if (nread == UV_EOF) { + //TODO check more when close + } else { + } + udfdUvHandleError(conn); + } +} + +void udfdOnNewConnection(uv_stream_t *server, int status) { + debugPrint("%s", "on new connection"); + if (status < 0) { + // TODO + return; + } + + uv_pipe_t *client = (uv_pipe_t *) taosMemoryMalloc(sizeof(uv_pipe_t)); + uv_pipe_init(loop, client, 0); + if (uv_accept(server, (uv_stream_t *) client) == 0) { + SUdfdUvConn *ctx = taosMemoryMalloc(sizeof(SUdfdUvConn)); + ctx->client = (uv_stream_t *) client; + ctx->inputBuf = 0; + ctx->inputLen = 0; + ctx->inputCap = 0; + client->data = ctx; + ctx->client = (uv_stream_t *) client; + uv_read_start((uv_stream_t *) client, udfdAllocBuffer, udfdPipeRead); + } else { + uv_close((uv_handle_t *) client, NULL); + } +} + +void removeListeningPipe(int sig) { + uv_fs_t req; + uv_fs_unlink(loop, &req, "udf.sock", NULL); + exit(0); +} + +int main() { + debugPrint("libuv version: %x", UV_VERSION_HEX); + + loop = uv_default_loop(); + uv_fs_t req; + uv_fs_unlink(loop, &req, "udf.sock", NULL); + + uv_pipe_t server; + uv_pipe_init(loop, &server, 0); + + signal(SIGINT, removeListeningPipe); + + int r; + if ((r = uv_pipe_bind(&server, "udf.sock"))) { + debugPrint("Bind error %s\n", uv_err_name(r)); + removeListeningPipe(0); + return 1; + } + if ((r = uv_listen((uv_stream_t *) &server, 128, udfdOnNewConnection))) { + debugPrint("Listen error %s", uv_err_name(r)); + return 2; + } + uv_run(loop, UV_RUN_DEFAULT); + uv_loop_close(loop); +} diff --git a/source/libs/function/test/runUdf.c b/source/libs/function/test/runUdf.c new file mode 100644 index 0000000000000000000000000000000000000000..bd742d23d07ba5ae1467dcfc3fb658c8cd9bc4ef --- /dev/null +++ b/source/libs/function/test/runUdf.c @@ -0,0 +1,46 @@ +#include +#include +#include + +#include "uv.h" +#include "os.h" +#include "tudf.h" + +int main(int argc, char *argv[]) { + startUdfService(); + uv_sleep(1000); + char path[256] = {0}; + size_t cwdSize = 256; + int err = uv_cwd(path, &cwdSize); + if (err != 0) { + fprintf(stderr, "err cwd: %s\n", uv_strerror(err)); + return err; + } + fprintf(stdout, "current working directory:%s\n", path); + strcat(path, "/libudf1.so"); + SUdfInfo udfInfo = {.udfName="udf1", .path=path}; + + UdfHandle handle; + setupUdf(&udfInfo, &handle); + + //char state[5000000] = "state"; + //char input[5000000] = "input"; + int dataSize = 500; + int callCount = 2; + if (argc > 1) dataSize = atoi(argv[1]); + if (argc > 2) callCount = atoi(argv[2]); + char *state = taosMemoryMalloc(dataSize); + char *input = taosMemoryMalloc(dataSize); + SUdfDataBlock blockInput = {.data = input, .size = dataSize}; + SUdfDataBlock blockOutput; + char* newState; + int32_t newStateSize; + for (int l = 0; l < callCount; ++l) { + callUdf(handle, 0, state, dataSize, blockInput, &newState, &newStateSize, &blockOutput); + } + taosMemoryFree(state); + taosMemoryFree(input); + teardownUdf(handle); + + stopUdfService(); +} diff --git a/source/libs/function/test/udf1.c b/source/libs/function/test/udf1.c new file mode 100644 index 0000000000000000000000000000000000000000..dc88e8cf3e194532a34414b9362711141b0b8a9d --- /dev/null +++ b/source/libs/function/test/udf1.c @@ -0,0 +1,21 @@ +#include +#include +#include + +#include "os.h" +#include "tudf.h" + +void udf1(int8_t step, char *state, int32_t stateSize, SUdfDataBlock input, + char **newState, int32_t *newStateSize, SUdfDataBlock *output) { + fprintf(stdout, "%s, step:%d\n", "udf function called", step); + char *newStateBuf = taosMemoryMalloc(stateSize); + memcpy(newStateBuf, state, stateSize); + *newState = newStateBuf; + *newStateSize = stateSize; + + char *outputBuf = taosMemoryMalloc(input.size); + memcpy(outputBuf, input.data, input.size); + output->data = outputBuf; + output->size = input.size; + return; +} diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 83410306eb7b3fdc4c56b1283219a8c6122acd37..61dd9523811d0eb5a0fa330f72fc4a350cf2e2c5 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -62,7 +62,7 @@ static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, Iterat int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { taosThreadOnce(&isInit, indexInit); - SIndex* sIdx = calloc(1, sizeof(SIndex)); + SIndex* sIdx = taosMemoryCalloc(1, sizeof(SIndex)); if (sIdx == NULL) { return -1; } @@ -115,8 +115,8 @@ void indexClose(SIndex* sIdx) { taosThreadMutexDestroy(&sIdx->mtx); indexTFileDestroy(sIdx->tindex); #endif - free(sIdx->path); - free(sIdx); + taosMemoryFree(sIdx->path); + taosMemoryFree(sIdx); return; } @@ -179,16 +179,16 @@ int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result EIndexOperatorType opera = multiQuerys->opera; int nQuery = taosArrayGetSize(multiQuerys->query); - char** fields = malloc(sizeof(char*) * nQuery); - char** keys = malloc(sizeof(char*) * nQuery); - int* types = malloc(sizeof(int) * nQuery); + char** fields = taosMemoryMalloc(sizeof(char*) * nQuery); + char** keys = taosMemoryMalloc(sizeof(char*) * nQuery); + int* types = taosMemoryMalloc(sizeof(int) * nQuery); for (int i = 0; i < nQuery; i++) { SIndexTermQuery* p = taosArrayGet(multiQuerys->query, i); SIndexTerm* term = p->field_value; - fields[i] = calloc(1, term->nKey + 1); - keys[i] = calloc(1, term->nVal + 1); + fields[i] = taosMemoryCalloc(1, term->nKey + 1); + keys[i] = taosMemoryCalloc(1, term->nVal + 1); memcpy(fields[i], term->key, term->nKey); memcpy(keys[i], term->val, term->nVal); @@ -203,12 +203,12 @@ int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result } for (int i = 0; i < nQuery; i++) { - free(fields[i]); - free(keys[i]); + taosMemoryFree(fields[i]); + taosMemoryFree(keys[i]); } - free(fields); - free(keys); - free(types); + taosMemoryFree(fields); + taosMemoryFree(keys); + taosMemoryFree(types); #endif #ifdef USE_INVERTED_INDEX @@ -258,7 +258,7 @@ void indexOptsDestroy(SIndexOpts* opts) { * */ SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) { - SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)malloc(sizeof(SIndexMultiTermQuery)); + SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)taosMemoryMalloc(sizeof(SIndexMultiTermQuery)); if (p == NULL) { return NULL; } @@ -272,7 +272,7 @@ void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery) { indexTermDestroy(p->term); } taosArrayDestroy(pQuery->query); - free(pQuery); + taosMemoryFree(pQuery); }; int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EIndexQueryType qType) { SIndexTermQuery q = {.qType = qType, .term = term}; @@ -282,7 +282,7 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EInde SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName, int32_t nColName, const char* colVal, int32_t nColVal) { - SIndexTerm* t = (SIndexTerm*)calloc(1, (sizeof(SIndexTerm))); + SIndexTerm* t = (SIndexTerm*)taosMemoryCalloc(1, (sizeof(SIndexTerm))); if (t == NULL) { return NULL; } @@ -291,19 +291,19 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy t->operType = oper; t->colType = colType; - t->colName = (char*)calloc(1, nColName + 1); + t->colName = (char*)taosMemoryCalloc(1, nColName + 1); memcpy(t->colName, colName, nColName); t->nColName = nColName; - t->colVal = (char*)calloc(1, nColVal + 1); + t->colVal = (char*)taosMemoryCalloc(1, nColVal + 1); memcpy(t->colVal, colVal, nColVal); t->nColVal = nColVal; return t; } void indexTermDestroy(SIndexTerm* p) { - free(p->colName); - free(p->colVal); - free(p); + taosMemoryFree(p->colName); + taosMemoryFree(p->colVal); + taosMemoryFree(p); } SIndexMultiTerm* indexMultiTermCreate() { return taosArrayInit(4, sizeof(SIndexTerm*)); } @@ -536,7 +536,7 @@ void iterateValueDestroy(IterateValue* value, bool destroy) { taosArrayClear(value->val); } } - free(value->colVal); + taosMemoryFree(value->colVal); value->colVal = NULL; } static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { @@ -572,7 +572,7 @@ static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { END: if (tw != NULL) { writerCtxDestroy(tw->ctx, true); - free(tw); + taosMemoryFree(tw); } return -1; } diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 2b4327b091897c12293c582041a7a5fbc4fbfeb5..bf907726bc974fbf7be7d2b074ba1c1320f38847 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -40,7 +40,7 @@ static bool indexCacheIteratorNext(Iterate* itera); static IterateValue* indexCacheIteratorGetValue(Iterate* iter); IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type) { - IndexCache* cache = calloc(1, sizeof(IndexCache)); + IndexCache* cache = taosMemoryCalloc(1, sizeof(IndexCache)); if (cache == NULL) { indexError("failed to create index cache"); return NULL; @@ -113,8 +113,8 @@ void indexCacheDestroySkiplist(SSkipList* slt) { SSkipListNode* node = tSkipListIterGet(iter); CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); if (ct != NULL) { - free(ct->colVal); - free(ct); + taosMemoryFree(ct->colVal); + taosMemoryFree(ct); } } tSkipListDestroyIter(iter); @@ -144,16 +144,16 @@ void indexCacheDestroy(void* cache) { } indexMemUnRef(pCache->mem); indexMemUnRef(pCache->imm); - free(pCache->colName); + taosMemoryFree(pCache->colName); taosThreadMutexDestroy(&pCache->mtx); taosThreadCondDestroy(&pCache->finished); - free(pCache); + taosMemoryFree(pCache); } Iterate* indexCacheIteratorCreate(IndexCache* cache) { - Iterate* iiter = calloc(1, sizeof(Iterate)); + Iterate* iiter = taosMemoryCalloc(1, sizeof(Iterate)); if (iiter == NULL) { return NULL; } @@ -179,7 +179,7 @@ void indexCacheIteratorDestroy(Iterate* iter) { } tSkipListDestroyIter(iter->iter); iterateValueDestroy(&iter->val, true); - free(iter); + taosMemoryFree(iter); } int indexCacheSchedToMerge(IndexCache* pCache) { @@ -221,7 +221,7 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { IndexCache* pCache = cache; indexCacheRef(pCache); // encode data - CacheTerm* ct = calloc(1, sizeof(CacheTerm)); + CacheTerm* ct = taosMemoryCalloc(1, sizeof(CacheTerm)); if (cache == NULL) { return -1; } @@ -230,7 +230,7 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { if (hasJson) { ct->colVal = indexPackJsonData(term); } else { - ct->colVal = (char*)calloc(1, sizeof(char) * (term->nColVal + 1)); + ct->colVal = (char*)taosMemoryCalloc(1, sizeof(char) * (term->nColVal + 1)); memcpy(ct->colVal, term->colVal, term->nColVal); } ct->version = atomic_add_fetch_32(&pCache->version, 1); @@ -323,7 +323,7 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTempResult* result } if (hasJson) { - tfree(p); + taosMemoryFreeClear(p); } indexMemUnRef(mem); @@ -365,7 +365,7 @@ void indexMemUnRef(MemTable* tbl) { if (ref == 0) { SSkipList* slt = tbl->mem; indexCacheDestroySkiplist(slt); - free(tbl); + taosMemoryFree(tbl); } } @@ -373,8 +373,8 @@ static void indexCacheTermDestroy(CacheTerm* ct) { if (ct == NULL) { return; } - free(ct->colVal); - free(ct); + taosMemoryFree(ct->colVal); + taosMemoryFree(ct); } static char* indexCacheTermGet(const void* pData) { CacheTerm* p = (CacheTerm*)pData; @@ -394,7 +394,7 @@ static int32_t indexCacheTermCompare(const void* l, const void* r) { static MemTable* indexInternalCacheCreate(int8_t type) { type = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? TSDB_DATA_TYPE_BINARY : type; - MemTable* tbl = calloc(1, sizeof(MemTable)); + MemTable* tbl = taosMemoryCalloc(1, sizeof(MemTable)); indexMemRef(tbl); if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { tbl->mem = tSkipListCreate(MAX_SKIP_LIST_LEVEL, type, MAX_INDEX_KEY_LEN, indexCacheTermCompare, SL_ALLOW_DUP_KEY, diff --git a/source/libs/index/src/index_comm.c b/source/libs/index/src/index_comm.c index 4f3cbaa4da3e295ceb3a743b22543faf81388752..4dea5fa0118cde2ee7c4a649ab6b57056cbace67 100644 --- a/source/libs/index/src/index_comm.c +++ b/source/libs/index/src/index_comm.c @@ -27,7 +27,7 @@ char* indexPackJsonData(SIndexTerm* itm) { uint8_t ty = INDEX_TYPE_GET_TYPE(itm->colType); int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1; - char* buf = (char*)calloc(1, sz); + char* buf = (char*)taosMemoryCalloc(1, sz); char* p = buf; memcpy(p, itm->colName, itm->nColName); diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 58d5a871cd4e481c341d52c39cd3a76290d6bb29..09f382bbdc818de55a4bc77650907b1e08c68e5e 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -30,7 +30,7 @@ static uint8_t fstPackDetla(FstCountingWriter* wrt, CompiledAddr nodeAddr, Compi } FstUnFinishedNodes* fstUnFinishedNodesCreate() { - FstUnFinishedNodes* nodes = malloc(sizeof(FstUnFinishedNodes)); + FstUnFinishedNodes* nodes = taosMemoryMalloc(sizeof(FstUnFinishedNodes)); if (nodes == NULL) { return NULL; } @@ -42,7 +42,7 @@ FstUnFinishedNodes* fstUnFinishedNodesCreate() { static void unFinishedNodeDestroyElem(void* elem) { FstBuilderNodeUnfinished* b = (FstBuilderNodeUnfinished*)elem; fstBuilderNodeDestroy(b->node); - free(b->last); + taosMemoryFree(b->last); b->last = NULL; } void fstUnFinishedNodesDestroy(FstUnFinishedNodes* nodes) { @@ -51,11 +51,11 @@ void fstUnFinishedNodesDestroy(FstUnFinishedNodes* nodes) { } taosArrayDestroyEx(nodes->stack, unFinishedNodeDestroyElem); - free(nodes); + taosMemoryFree(nodes); } void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes* nodes, bool isFinal) { - FstBuilderNode* node = malloc(sizeof(FstBuilderNode)); + FstBuilderNode* node = taosMemoryMalloc(sizeof(FstBuilderNode)); node->isFinal = isFinal; node->finalOutput = 0; node->trans = taosArrayInit(16, sizeof(FstTransition)); @@ -74,7 +74,7 @@ FstBuilderNode* fstUnFinishedNodesPopRoot(FstUnFinishedNodes* nodes) { FstBuilderNode* fstUnFinishedNodesPopFreeze(FstUnFinishedNodes* nodes, CompiledAddr addr) { FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); fstBuilderNodeUnfinishedLastCompiled(un, addr); - // free(un->last); // TODO add func FstLastTransitionFree() + // taosMemoryFree(un->last); // TODO add func FstLastTransitionFree() // un->last = NULL; return un->node; } @@ -103,7 +103,7 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, sz); assert(un->last == NULL); - // FstLastTransition *trn = malloc(sizeof(FstLastTransition)); + // FstLastTransition *trn = taosMemoryMalloc(sizeof(FstLastTransition)); // trn->inp = s->data[s->start]; // trn->out = out; int32_t len = 0; @@ -111,12 +111,12 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output un->last = fstLastTransitionCreate(data[0], out); for (uint64_t i = 1; i < len; i++) { - FstBuilderNode* n = malloc(sizeof(FstBuilderNode)); + FstBuilderNode* n = taosMemoryMalloc(sizeof(FstBuilderNode)); n->isFinal = false; n->finalOutput = 0; n->trans = taosArrayInit(16, sizeof(FstTransition)); - // FstLastTransition *trn = malloc(sizeof(FstLastTransition)); + // FstLastTransition *trn = taosMemoryMalloc(sizeof(FstLastTransition)); // trn->inp = s->data[i]; // trn->out = out; FstLastTransition* trn = fstLastTransitionCreate(data[i], 0); @@ -296,7 +296,7 @@ void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuil // at that index. (Except when there are 256 transitions.) Namely, // any value greater than or equal to the number of transitions in // this node indicates an absent transition. - uint8_t* index = (uint8_t*)malloc(sizeof(uint8_t) * 256); + uint8_t* index = (uint8_t*)taosMemoryMalloc(sizeof(uint8_t) * 256); memset(index, 255, sizeof(uint8_t) * 256); /// for (uint8_t i = 0; i < 256; i++) { // index[i] = 255; @@ -307,7 +307,7 @@ void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuil // fstPackDeltaIn(w, addr, t->addr, tSize); } fstCountingWriterWrite(w, (char*)index, 256); - free(index); + taosMemoryFree(index); } fstCountingWriterWrite(w, (char*)&packSizes, 1); bool null = false; @@ -578,7 +578,7 @@ uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) { // fst node function FstNode* fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice* slice) { - FstNode* n = (FstNode*)malloc(sizeof(FstNode)); + FstNode* n = (FstNode*)taosMemoryMalloc(sizeof(FstNode)); if (n == NULL) { return NULL; } @@ -643,10 +643,10 @@ static const char* fstNodeState(FstNode* node) { void fstNodeDestroy(FstNode* node) { fstSliceDestroy(&node->data); - free(node); + taosMemoryFree(node); } FstTransitions* fstNodeTransitions(FstNode* node) { - FstTransitions* t = malloc(sizeof(FstTransitions)); + FstTransitions* t = taosMemoryMalloc(sizeof(FstTransitions)); if (NULL == t) { return NULL; } @@ -755,7 +755,7 @@ bool fstBuilderNodeCompileTo(FstBuilderNode* b, FstCountingWriter* wrt, Compiled } FstBuilder* fstBuilderCreate(void* w, FstType ty) { - FstBuilder* b = malloc(sizeof(FstBuilder)); + FstBuilder* b = taosMemoryMalloc(sizeof(FstBuilder)); if (NULL == b) { return b; } @@ -788,7 +788,7 @@ void fstBuilderDestroy(FstBuilder* b) { fstUnFinishedNodesDestroy(b->unfinished); fstRegistryDestroy(b->registry); fstSliceDestroy(&b->last); - free(b); + taosMemoryFree(b); } bool fstBuilderInsert(FstBuilder* b, FstSlice bs, Output in) { @@ -928,7 +928,7 @@ FstSlice fstNodeAsSlice(FstNode* node) { } FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) { - FstLastTransition* trn = malloc(sizeof(FstLastTransition)); + FstLastTransition* trn = taosMemoryMalloc(sizeof(FstLastTransition)); if (trn == NULL) { return NULL; } @@ -938,7 +938,7 @@ FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) { return trn; } -void fstLastTransitionDestroy(FstLastTransition* trn) { free(trn); } +void fstLastTransitionDestroy(FstLastTransition* trn) { taosMemoryFree(trn); } void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, CompiledAddr addr) { FstLastTransition* trn = unNode->last; @@ -1003,12 +1003,12 @@ Fst* fstCreate(FstSlice* slice) { len -= sizeof(fstLen); taosDecodeFixedU64(buf + len, &fstLen); // TODO(validate root addr) - Fst* fst = (Fst*)calloc(1, sizeof(Fst)); + Fst* fst = (Fst*)taosMemoryCalloc(1, sizeof(Fst)); if (fst == NULL) { return NULL; } - fst->meta = (FstMeta*)malloc(sizeof(FstMeta)); + fst->meta = (FstMeta*)taosMemoryMalloc(sizeof(FstMeta)); if (NULL == fst->meta) { goto FST_CREAT_FAILED; } @@ -1019,7 +1019,7 @@ Fst* fstCreate(FstSlice* slice) { fst->meta->len = fstLen; fst->meta->checkSum = checkSum; - FstSlice* s = calloc(1, sizeof(FstSlice)); + FstSlice* s = taosMemoryCalloc(1, sizeof(FstSlice)); *s = fstSliceCopy(slice, 0, FST_SLICE_LEN(slice) - 1); fst->data = s; @@ -1027,19 +1027,19 @@ Fst* fstCreate(FstSlice* slice) { return fst; FST_CREAT_FAILED: - free(fst->meta); - free(fst); + taosMemoryFree(fst->meta); + taosMemoryFree(fst); return NULL; } void fstDestroy(Fst* fst) { if (fst) { - free(fst->meta); + taosMemoryFree(fst->meta); fstSliceDestroy(fst->data); - free(fst->data); + taosMemoryFree(fst->data); taosThreadMutexDestroy(&fst->mtx); } - free(fst); + taosMemoryFree(fst); } bool fstGet(Fst* fst, FstSlice* b, Output* out) { @@ -1136,7 +1136,7 @@ bool fstVerify(Fst* fst) { // data bound function FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice* data) { - FstBoundWithData* b = calloc(1, sizeof(FstBoundWithData)); + FstBoundWithData* b = taosMemoryCalloc(1, sizeof(FstBoundWithData)); if (b == NULL) { return NULL; } @@ -1171,11 +1171,11 @@ bool fstBoundWithDataIsEmpty(FstBoundWithData* bound) { bool fstBoundWithDataIsIncluded(FstBoundWithData* bound) { return bound->type == Excluded ? false : true; } -void fstBoundDestroy(FstBoundWithData* bound) { free(bound); } +void fstBoundDestroy(FstBoundWithData* bound) { taosMemoryFree(bound); } StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, FstBoundWithData* max) { - StreamWithState* sws = calloc(1, sizeof(StreamWithState)); + StreamWithState* sws = taosMemoryCalloc(1, sizeof(StreamWithState)); if (sws == NULL) { return NULL; } @@ -1201,7 +1201,7 @@ void streamWithStateDestroy(StreamWithState* sws) { taosArrayDestroy(sws->inp); taosArrayDestroyEx(sws->stack, streamStateDestroy); - free(sws); + taosMemoryFree(sws); } bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) { @@ -1346,7 +1346,7 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb taosArrayPush(sws->stack, &s2); size_t isz = taosArrayGetSize(sws->inp); - uint8_t* buf = (uint8_t*)malloc(isz * sizeof(uint8_t)); + uint8_t* buf = (uint8_t*)taosMemoryMalloc(isz * sizeof(uint8_t)); for (uint32_t i = 0; i < isz; i++) { buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i); } @@ -1354,19 +1354,19 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb if (fstBoundWithDataExceededBy(sws->endAt, &slice)) { taosArrayDestroyEx(sws->stack, streamStateDestroy); sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); - tfree(buf); + taosMemoryFreeClear(buf); fstSliceDestroy(&slice); return NULL; } if (FST_NODE_IS_FINAL(nextNode) && isMatch) { FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)}; StreamWithStateResult* result = swsResultCreate(&slice, fOutput, tState); - tfree(buf); + taosMemoryFreeClear(buf); fstSliceDestroy(&slice); taosArrayDestroy(nodes); return result; } - tfree(buf); + taosMemoryFreeClear(buf); fstSliceDestroy(&slice); } for (size_t i = 0; i < taosArrayGetSize(nodes); i++) { @@ -1378,7 +1378,7 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb } StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state) { - StreamWithStateResult* result = calloc(1, sizeof(StreamWithStateResult)); + StreamWithStateResult* result = taosMemoryCalloc(1, sizeof(StreamWithStateResult)); if (result == NULL) { return NULL; } @@ -1395,7 +1395,7 @@ void swsResultDestroy(StreamWithStateResult* result) { fstSliceDestroy(&result->data); startWithStateValueDestroy(result->state); - free(result); + taosMemoryFree(result); } void streamStateDestroy(void* s) { @@ -1407,7 +1407,7 @@ void streamStateDestroy(void* s) { } FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) { - FstStreamBuilder* b = calloc(1, sizeof(FstStreamBuilder)); + FstStreamBuilder* b = taosMemoryCalloc(1, sizeof(FstStreamBuilder)); if (NULL == b) { return NULL; } @@ -1421,9 +1421,9 @@ FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) { void fstStreamBuilderDestroy(FstStreamBuilder* b) { fstSliceDestroy(&b->min->data); fstSliceDestroy(&b->max->data); - tfree(b->min); - tfree(b->max); - free(b); + taosMemoryFreeClear(b->min); + taosMemoryFreeClear(b->max); + taosMemoryFree(b); } FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, RangeType type) { if (b == NULL) { diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index ed1ad7a374286c8251dcb719be9a5510c6eec844..668a527d4a56e675f0b4ddc1a1393c74e20f7732 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -16,7 +16,7 @@ #include "index_fst_automation.h" StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val) { - StartWithStateValue* sv = calloc(1, sizeof(StartWithStateValue)); + StartWithStateValue* sv = taosMemoryCalloc(1, sizeof(StartWithStateValue)); if (sv == NULL) { return NULL; } @@ -27,7 +27,7 @@ StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueTyp sv->val = *(int*)val; } else if (ty == FST_CHAR) { size_t len = strlen((char*)val); - sv->ptr = (char*)calloc(1, len + 1); + sv->ptr = (char*)taosMemoryCalloc(1, len + 1); memcpy(sv->ptr, val, len); } else if (ty == FST_ARRAY) { // TODO, @@ -44,14 +44,14 @@ void startWithStateValueDestroy(void* val) { if (sv->type == FST_INT) { // } else if (sv->type == FST_CHAR) { - free(sv->ptr); + taosMemoryFree(sv->ptr); } else if (sv->type == FST_ARRAY) { taosArrayDestroy(sv->arr); } - free(sv); + taosMemoryFree(sv); } StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) { - StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); + StartWithStateValue* nsv = taosMemoryCalloc(1, sizeof(StartWithStateValue)); if (nsv == NULL) { return NULL; } @@ -62,7 +62,7 @@ StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) { nsv->val = sv->val; } else if (nsv->type == FST_CHAR) { size_t len = strlen(sv->ptr); - nsv->ptr = (char*)calloc(1, len + 1); + nsv->ptr = (char*)taosMemoryCalloc(1, len + 1); memcpy(nsv->ptr, sv->ptr, len); } else if (nsv->type == FST_ARRAY) { // @@ -137,7 +137,7 @@ AutomationFunc automFuncs[] = { }; AutomationCtx* automCtxCreate(void* data, AutomationType atype) { - AutomationCtx* ctx = calloc(1, sizeof(AutomationCtx)); + AutomationCtx* ctx = taosMemoryCalloc(1, sizeof(AutomationCtx)); if (ctx == NULL) { return NULL; } @@ -158,7 +158,7 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) { if (data != NULL) { char* src = (char*)data; size_t len = strlen(src); - dst = (char*)calloc(1, len * sizeof(char) + 1); + dst = (char*)taosMemoryCalloc(1, len * sizeof(char) + 1); memcpy(dst, src, len); } @@ -169,6 +169,6 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) { } void automCtxDestroy(AutomationCtx* ctx) { startWithStateValueDestroy(ctx->stdata); - free(ctx->data); - free(ctx); + taosMemoryFree(ctx->data); + taosMemoryFree(ctx); } diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index 8cb2ff92461382fc2549f792c2cd326650946f04..6a161ba7e954085f753e0137b41fb98e2dc2b230 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -81,7 +81,7 @@ static int writeCtxDoFlush(WriterCtx* ctx) { } WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int32_t capacity) { - WriterCtx* ctx = calloc(1, sizeof(WriterCtx)); + WriterCtx* ctx = taosMemoryCalloc(1, sizeof(WriterCtx)); if (ctx == NULL) { return NULL; } ctx->type = type; @@ -112,7 +112,7 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int goto END; } } else if (ctx->type == TMemory) { - ctx->mem.buf = calloc(1, sizeof(char) * capacity); + ctx->mem.buf = taosMemoryCalloc(1, sizeof(char) * capacity); ctx->mem.capa = capacity; } ctx->write = writeCtxDoWrite; @@ -126,13 +126,13 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int return ctx; END: - if (ctx->type == TMemory) { free(ctx->mem.buf); } - free(ctx); + if (ctx->type == TMemory) { taosMemoryFree(ctx->mem.buf); } + taosMemoryFree(ctx); return NULL; } void writerCtxDestroy(WriterCtx* ctx, bool remove) { if (ctx->type == TMemory) { - free(ctx->mem.buf); + taosMemoryFree(ctx->mem.buf); } else { ctx->flush(ctx); taosCloseFile(&ctx->file.pFile); @@ -150,11 +150,11 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) { } if (remove) { unlink(ctx->file.buf); } } - free(ctx); + taosMemoryFree(ctx); } FstCountingWriter* fstCountingWriterCreate(void* wrt) { - FstCountingWriter* cw = calloc(1, sizeof(FstCountingWriter)); + FstCountingWriter* cw = taosMemoryCalloc(1, sizeof(FstCountingWriter)); if (cw == NULL) { return NULL; } cw->wrt = wrt; @@ -165,7 +165,7 @@ void fstCountingWriterDestroy(FstCountingWriter* cw) { // free wrt object: close fd or free mem fstCountingWriterFlush(cw); // writerCtxDestroy((WriterCtx *)(cw->wrt)); - free(cw); + taosMemoryFree(cw); } int fstCountingWriterWrite(FstCountingWriter* write, uint8_t* buf, uint32_t len) { @@ -203,13 +203,13 @@ int fstCountingWriterFlush(FstCountingWriter* write) { void fstCountingWriterPackUintIn(FstCountingWriter* writer, uint64_t n, uint8_t nBytes) { assert(1 <= nBytes && nBytes <= 8); - uint8_t* buf = calloc(8, sizeof(uint8_t)); + uint8_t* buf = taosMemoryCalloc(8, sizeof(uint8_t)); for (uint8_t i = 0; i < nBytes; i++) { buf[i] = (uint8_t)n; n = n >> 8; } fstCountingWriterWrite(writer, buf, nBytes); - free(buf); + taosMemoryFree(buf); return; } diff --git a/source/libs/index/src/index_fst_node.c b/source/libs/index/src/index_fst_node.c index 5cdf6adb31a421e38fb22d30fe34599f1489924c..a0d59150d73231cddd2781ad6627e5d29c5e0422 100644 --- a/source/libs/index/src/index_fst_node.c +++ b/source/libs/index/src/index_fst_node.c @@ -15,7 +15,7 @@ #include "index_fst_node.h" FstBuilderNode* fstBuilderNodeDefault() { - FstBuilderNode* bn = malloc(sizeof(FstBuilderNode)); + FstBuilderNode* bn = taosMemoryMalloc(sizeof(FstBuilderNode)); bn->isFinal = false; bn->finalOutput = 0; bn->trans = taosArrayInit(16, sizeof(FstTransition)); @@ -25,7 +25,7 @@ void fstBuilderNodeDestroy(FstBuilderNode* node) { if (node == NULL) { return; } taosArrayDestroy(node->trans); - free(node); + taosMemoryFree(node); } bool fstBuilderNodeEqual(FstBuilderNode* n1, FstBuilderNode* n2) { @@ -45,7 +45,7 @@ bool fstBuilderNodeEqual(FstBuilderNode* n1, FstBuilderNode* n2) { return true; } FstBuilderNode* fstBuilderNodeClone(FstBuilderNode* src) { - FstBuilderNode* node = malloc(sizeof(FstBuilderNode)); + FstBuilderNode* node = taosMemoryMalloc(sizeof(FstBuilderNode)); if (node == NULL) { return NULL; } // diff --git a/source/libs/index/src/index_fst_registry.c b/source/libs/index/src/index_fst_registry.c index b28b518fc156b5d7585a2e071b603a186bcd54e7..7b6b8df25f5b7ad85d3d181a3a9ba6b9d8c9fbe8 100644 --- a/source/libs/index/src/index_fst_registry.c +++ b/source/libs/index/src/index_fst_registry.c @@ -65,13 +65,13 @@ static void fstRegistryCellPromote(SArray* arr, uint32_t start, uint32_t end) { } FstRegistry* fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) { - FstRegistry* registry = malloc(sizeof(FstRegistry)); + FstRegistry* registry = taosMemoryMalloc(sizeof(FstRegistry)); if (registry == NULL) { return NULL; } uint64_t nCells = tableSize * mruSize; SArray* tb = (SArray*)taosArrayInit(nCells, sizeof(FstRegistryCell)); if (NULL == tb) { - free(registry); + taosMemoryFree(registry); return NULL; } @@ -96,7 +96,7 @@ void fstRegistryDestroy(FstRegistry* registry) { fstBuilderNodeDestroy(cell->node); } taosArrayDestroy(tb); - free(registry); + taosMemoryFree(registry); } FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNode) { @@ -105,7 +105,7 @@ FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNo uint64_t start = registry->mruSize * bucket; uint64_t end = start + registry->mruSize; - FstRegistryEntry* entry = malloc(sizeof(FstRegistryEntry)); + FstRegistryEntry* entry = taosMemoryMalloc(sizeof(FstRegistryEntry)); if (end - start == 1) { FstRegistryCell* cell = taosArrayGet(registry->table, start); // cell->isNode && @@ -166,5 +166,5 @@ FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNo return entry; } void fstRegistryEntryDestroy(FstRegistryEntry* entry) { - free(entry); + taosMemoryFree(entry); } diff --git a/source/libs/index/src/index_fst_util.c b/source/libs/index/src/index_fst_util.c index f08a48c34ed6040b21b98f62e0514f0d0664c2bb..f9581f7202dc127d62f09ca5bf8b67f3ff4341b2 100644 --- a/source/libs/index/src/index_fst_util.c +++ b/source/libs/index/src/index_fst_util.c @@ -78,10 +78,10 @@ CompiledAddr unpackDelta(char* data, uint64_t len, uint64_t nodeAddr) { // FstSlice fstSliceCreate(uint8_t* data, uint64_t len) { - FstString* str = (FstString*)malloc(sizeof(FstString)); + FstString* str = (FstString*)taosMemoryMalloc(sizeof(FstString)); str->ref = 1; str->len = len; - str->data = malloc(len * sizeof(uint8_t)); + str->data = taosMemoryMalloc(len * sizeof(uint8_t)); memcpy(str->data, data, len); FstSlice s = {.str = str, .start = 0, .end = len - 1}; @@ -101,10 +101,10 @@ FstSlice fstSliceDeepCopy(FstSlice* s, int32_t start, int32_t end) { uint8_t* data = fstSliceData(s, &slen); assert(tlen <= slen); - uint8_t* buf = malloc(sizeof(uint8_t) * tlen); + uint8_t* buf = taosMemoryMalloc(sizeof(uint8_t) * tlen); memcpy(buf, data + start, tlen); - FstString* str = malloc(sizeof(FstString)); + FstString* str = taosMemoryMalloc(sizeof(FstString)); str->data = buf; str->len = tlen; str->ref = 1; @@ -128,8 +128,8 @@ void fstSliceDestroy(FstSlice* s) { FstString* str = s->str; str->ref--; if (str->ref == 0) { - free(str->data); - free(str); + taosMemoryFree(str->data); + taosMemoryFree(str); s->str = NULL; } } @@ -161,7 +161,7 @@ int fstSliceCompare(FstSlice* a, FstSlice* b) { } // FstStack* fstStackCreate(size_t elemSize, StackFreeElem freeFn) { -// FstStack *s = calloc(1, sizeof(FstStack)); +// FstStack *s = taosMemoryCalloc(1, sizeof(FstStack)); // if (s == NULL) { return NULL; } // s-> // s->freeFn diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 780b7160fca098a851071871d5e79b3e3768a57d..53813e13e6920cacd435ef9ebc65525a2c80760d 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -59,7 +59,7 @@ static void tfileGenFileName(char* filename, uint64_t suid, const char* col, static void tfileGenFileFullName(char* fullname, const char* path, uint64_t suid, const char* col, int32_t version); TFileCache* tfileCacheCreate(const char* path) { - TFileCache* tcache = calloc(1, sizeof(TFileCache)); + TFileCache* tcache = taosMemoryCalloc(1, sizeof(TFileCache)); if (tcache == NULL) { return NULL; } @@ -113,7 +113,7 @@ void tfileCacheDestroy(TFileCache* tcache) { reader = taosHashIterate(tcache->tableCache, reader); } taosHashCleanup(tcache->tableCache); - free(tcache); + taosMemoryFree(tcache); } TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key) { @@ -145,7 +145,7 @@ void tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader) { return; } TFileReader* tfileReaderCreate(WriterCtx* ctx) { - TFileReader* reader = calloc(1, sizeof(TFileReader)); + TFileReader* reader = taosMemoryCalloc(1, sizeof(TFileReader)); if (reader == NULL) { return NULL; } @@ -181,7 +181,7 @@ void tfileReaderDestroy(TFileReader* reader) { // T_REF_INC(reader); fstDestroy(reader->fst); writerCtxDestroy(reader->ctx, reader->remove); - free(reader); + taosMemoryFree(reader); } int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTempResult* tr) { @@ -218,7 +218,7 @@ int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTempResul } fstSliceDestroy(&key); if (hasJson) { - free(p); + taosMemoryFree(p); } } else if (qtype == QUERY_PREFIX) { // handle later @@ -269,7 +269,7 @@ TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const c return reader; } TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) { - TFileWriter* tw = calloc(1, sizeof(TFileWriter)); + TFileWriter* tw = taosMemoryCalloc(1, sizeof(TFileWriter)); if (tw == NULL) { indexError("index: %" PRIu64 " failed to alloc TFilerWriter", header->suid); return NULL; @@ -296,7 +296,7 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { } int32_t bufLimit = 64 * 4096, offset = 0; - // char* buf = calloc(1, sizeof(char) * bufLimit); + // char* buf = taosMemoryCalloc(1, sizeof(char) * bufLimit); // char* p = buf; int32_t sz = taosArrayGetSize((SArray*)data); int32_t fstOffset = tw->offset; @@ -318,13 +318,13 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { // check buf has enough space or not int32_t ttsz = TF_TABLE_TATOAL_SIZE(tbsz); - char* buf = calloc(1, ttsz * sizeof(char)); + char* buf = taosMemoryCalloc(1, ttsz * sizeof(char)); char* p = buf; tfileSerialTableIdsToBuf(p, v->tableId); tw->ctx->write(tw->ctx, buf, ttsz); v->offset = tw->offset; tw->offset += ttsz; - free(buf); + taosMemoryFree(buf); } tw->fb = fstBuilderCreate(tw->ctx, 0); @@ -359,14 +359,14 @@ void tfileWriterClose(TFileWriter* tw) { return; } writerCtxDestroy(tw->ctx, false); - free(tw); + taosMemoryFree(tw); } void tfileWriterDestroy(TFileWriter* tw) { if (tw == NULL) { return; } writerCtxDestroy(tw->ctx, false); - free(tw); + taosMemoryFree(tw); } IndexTFile* indexTFileCreate(const char* path) { @@ -375,7 +375,7 @@ IndexTFile* indexTFileCreate(const char* path) { return NULL; } - IndexTFile* tfile = calloc(1, sizeof(IndexTFile)); + IndexTFile* tfile = taosMemoryCalloc(1, sizeof(IndexTFile)); if (tfile == NULL) { tfileCacheDestroy(cache); return NULL; @@ -389,7 +389,7 @@ void indexTFileDestroy(IndexTFile* tfile) { return; } tfileCacheDestroy(tfile->cache); - free(tfile); + taosMemoryFree(tfile); } int indexTFileSearch(void* tfile, SIndexTermQuery* query, SIdxTempResult* result) { @@ -433,7 +433,7 @@ static bool tfileIteratorNext(Iterate* iiter) { int32_t sz = 0; char* ch = (char*)fstSliceData(&rt->data, &sz); - colVal = calloc(1, sz + 1); + colVal = taosMemoryCalloc(1, sz + 1); memcpy(colVal, ch, sz); offset = (uint64_t)(rt->out.out); @@ -453,7 +453,7 @@ static bool tfileIteratorNext(Iterate* iiter) { static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; } static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) { - TFileFstIter* tIter = calloc(1, sizeof(TFileFstIter)); + TFileFstIter* tIter = taosMemoryCalloc(1, sizeof(TFileFstIter)); if (tIter == NULL) { return NULL; } @@ -470,10 +470,10 @@ Iterate* tfileIteratorCreate(TFileReader* reader) { return NULL; } - Iterate* iter = calloc(1, sizeof(Iterate)); + Iterate* iter = taosMemoryCalloc(1, sizeof(Iterate)); iter->iter = tfileFstIteratorCreate(reader); if (iter->iter == NULL) { - free(iter); + taosMemoryFree(iter); return NULL; } iter->next = tfileIteratorNext; @@ -494,9 +494,9 @@ void tfileIteratorDestroy(Iterate* iter) { streamWithStateDestroy(tIter->st); fstStreamBuilderDestroy(tIter->fb); automCtxDestroy(tIter->ctx); - free(tIter); + taosMemoryFree(tIter); - free(iter); + taosMemoryFree(iter); } TFileReader* tfileGetReaderByCol(IndexTFile* tf, uint64_t suid, char* colName) { @@ -530,7 +530,7 @@ static int tfileValueCompare(const void* a, const void* b, const void* param) { } TFileValue* tfileValueCreate(char* val) { - TFileValue* tf = calloc(1, sizeof(TFileValue)); + TFileValue* tf = taosMemoryCalloc(1, sizeof(TFileValue)); if (tf == NULL) { return NULL; } @@ -547,8 +547,8 @@ int tfileValuePush(TFileValue* tf, uint64_t val) { } void tfileValueDestroy(TFileValue* tf) { taosArrayDestroy(tf->tableId); - free(tf->colVal); - free(tf); + taosMemoryFree(tf->colVal); + taosMemoryFree(tf); } static void tfileSerialTableIdsToBuf(char* buf, SArray* ids) { int sz = taosArrayGetSize(ids); @@ -636,7 +636,7 @@ static int tfileReaderLoadFst(TFileReader* reader) { // current load fst into memory, refactor it later int fstSize = size - reader->header.fstOffset - sizeof(tfileMagicNumber); - char* buf = calloc(1, fstSize); + char* buf = taosMemoryCalloc(1, fstSize); if (buf == NULL) { return -1; } @@ -651,7 +651,7 @@ static int tfileReaderLoadFst(TFileReader* reader) { FstSlice st = fstSliceCreate((uint8_t*)buf, nread); reader->fst = fstCreate(&st); - free(buf); + taosMemoryFree(buf); fstSliceDestroy(&st); return reader->fst != NULL ? 0 : -1; @@ -744,7 +744,7 @@ static SArray* tfileGetFileList(const char* path) { } size_t len = strlen(path) + 1 + strlen(file) + 1; - char* buf = calloc(1, len); + char* buf = taosMemoryCalloc(1, len); sprintf(buf, "%s/%s", path, file); taosArrayPush(files, &buf); } @@ -761,7 +761,7 @@ static int tfileRmExpireFile(SArray* result) { } static void tfileDestroyFileName(void* elem) { char* p = *(char**)elem; - free(p); + taosMemoryFree(p); } static int tfileCompare(const void* a, const void* b) { const char* as = *(char**)a; diff --git a/source/libs/index/src/index_util.c b/source/libs/index/src/index_util.c index dfe4e273a9e09255d2016b874e76ed108fc011c4..65c16ca65ba5b54de2f3492abe4314a6419c4e16 100644 --- a/source/libs/index/src/index_util.c +++ b/source/libs/index/src/index_util.c @@ -41,7 +41,7 @@ void iIntersection(SArray *inters, SArray *final) { if (sz <= 0) { return; } - MergeIndex *mi = calloc(sz, sizeof(MergeIndex)); + MergeIndex *mi = taosMemoryCalloc(sz, sizeof(MergeIndex)); for (int i = 0; i < sz; i++) { SArray *t = taosArrayGetP(inters, i); mi[i].len = taosArrayGetSize(t); @@ -67,7 +67,7 @@ void iIntersection(SArray *inters, SArray *final) { taosArrayPush(final, &tgt); } } - tfree(mi); + taosMemoryFreeClear(mi); } void iUnion(SArray *inters, SArray *final) { int32_t sz = taosArrayGetSize(inters); @@ -79,7 +79,7 @@ void iUnion(SArray *inters, SArray *final) { return; } - MergeIndex *mi = calloc(sz, sizeof(MergeIndex)); + MergeIndex *mi = taosMemoryCalloc(sz, sizeof(MergeIndex)); for (int i = 0; i < sz; i++) { SArray *t = taosArrayGetP(inters, i); mi[i].len = taosArrayGetSize(t); @@ -113,7 +113,7 @@ void iUnion(SArray *inters, SArray *final) { break; } } - tfree(mi); + taosMemoryFreeClear(mi); } void iExcept(SArray *total, SArray *except) { @@ -156,7 +156,7 @@ int verdataCompare(const void *a, const void *b) { } SIdxTempResult *sIdxTempResultCreate() { - SIdxTempResult *tr = calloc(1, sizeof(SIdxTempResult)); + SIdxTempResult *tr = taosMemoryCalloc(1, sizeof(SIdxTempResult)); tr->total = taosArrayInit(4, sizeof(uint64_t)); tr->added = taosArrayInit(4, sizeof(uint64_t)); diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index 410d5b702125df60c780fbcf5a90951dc9d24323..94923726ddb4667be7352786916135739bd4ed13 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -55,7 +55,7 @@ class FstReadMemory { memset((void*)&_s, 0, sizeof(_s)); } bool init() { - char* buf = (char*)calloc(1, sizeof(char) * _size); + char* buf = (char*)taosMemoryCalloc(1, sizeof(char) * _size); int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size); if (nRead <= 0) { return false; @@ -63,7 +63,7 @@ class FstReadMemory { _size = nRead; _s = fstSliceCreate((uint8_t*)buf, _size); _fst = fstCreate(&_s); - free(buf); + taosMemoryFree(buf); return _fst != NULL; } bool Get(const std::string& key, uint64_t* val) { @@ -230,7 +230,7 @@ void checkFstLongTerm() { // for (int i = 0; i < result.size(); i++) { // assert(result[i] == i); // check result //} - // free(ctx); + // taosMemoryFree(ctx); // delete m; } void checkFstCheckIterator() { @@ -266,7 +266,7 @@ void checkFstCheckIterator() { // assert(result[i] == i); // check result } - free(ctx); + taosMemoryFree(ctx); delete m; } diff --git a/source/libs/index/test/fstUT.cc b/source/libs/index/test/fstUT.cc index 6e9c88bc7953c4d7d2b6e8b142198b606290dc4d..1bdc7fc9c9a0c4f2081c6f36038fee47c5fd3338 100644 --- a/source/libs/index/test/fstUT.cc +++ b/source/libs/index/test/fstUT.cc @@ -75,7 +75,7 @@ class FstReadMemory { memset((void*)&_s, 0, sizeof(_s)); } bool init() { - char* buf = (char*)calloc(1, sizeof(char) * _size); + char* buf = (char*)taosMemoryCalloc(1, sizeof(char) * _size); int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size); if (nRead <= 0) { return false; @@ -83,7 +83,7 @@ class FstReadMemory { _size = nRead; _s = fstSliceCreate((uint8_t*)buf, _size); _fst = fstCreate(&_s); - free(buf); + taosMemoryFreeClear(buf); return _fst != NULL; } bool Get(const std::string& key, uint64_t* val) { diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 5f339f286576f3fd5d535470ed9583a2b65363df..0e4eb060cf06e8414cd41b048ff44fdc7feab751 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -81,7 +81,7 @@ class FstReadMemory { memset((void*)&_s, 0, sizeof(_s)); } bool init() { - char* buf = (char*)calloc(1, sizeof(char) * _size); + char* buf = (char*)taosMemoryCalloc(1, sizeof(char) * _size); int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size); if (nRead <= 0) { return false; @@ -89,7 +89,7 @@ class FstReadMemory { _size = nRead; _s = fstSliceCreate((uint8_t*)buf, _size); _fst = fstCreate(&_s); - free(buf); + taosMemoryFree(buf); return _fst != NULL; } bool Get(const std::string& key, uint64_t* val) { @@ -227,7 +227,7 @@ void checkFstPrefixSearch() { assert(result[i] == i); // check result } - free(ctx); + taosMemoryFree(ctx); delete m; } void validateFst() { @@ -446,9 +446,9 @@ class IndexTFileEnv : public ::testing::Test { }; static TFileValue* genTFileValue(const char* val) { - TFileValue* tv = (TFileValue*)calloc(1, sizeof(TFileValue)); + TFileValue* tv = (TFileValue*)taosMemoryCalloc(1, sizeof(TFileValue)); int32_t vlen = strlen(val) + 1; - tv->colVal = (char*)calloc(1, vlen); + tv->colVal = (char*)taosMemoryCalloc(1, vlen); memcpy(tv->colVal, val, vlen); tv->tableId = (SArray*)taosArrayInit(1, sizeof(uint64_t)); @@ -460,9 +460,9 @@ static TFileValue* genTFileValue(const char* val) { } static void destroyTFileValue(void* val) { TFileValue* tv = (TFileValue*)val; - free(tv->colVal); + taosMemoryFree(tv->colVal); taosArrayDestroy(tv->tableId); - free(tv); + taosMemoryFree(tv); } TEST_F(IndexTFileEnv, test_tfile_write) { TFileValue* v1 = genTFileValue("ab"); diff --git a/source/libs/monitor/src/monitor.c b/source/libs/monitor/src/monitor.c index aa5eafd8b23de6167ad1980511a66f2e3f884d81..b92c08d51cbc23b9a22cd8a0f56766cb2852b226 100644 --- a/source/libs/monitor/src/monitor.c +++ b/source/libs/monitor/src/monitor.c @@ -60,7 +60,7 @@ void monCleanup() { } SMonInfo *monCreateMonitorInfo() { - SMonInfo *pMonitor = calloc(1, sizeof(SMonInfo)); + SMonInfo *pMonitor = taosMemoryCalloc(1, sizeof(SMonInfo)); if (pMonitor == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -88,7 +88,7 @@ void monCleanupMonitorInfo(SMonInfo *pMonitor) { tsMonitor.state.time = pMonitor->curTime; taosArrayDestroy(pMonitor->logs); tjsonDelete(pMonitor->pJson); - free(pMonitor); + taosMemoryFree(pMonitor); } void monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo) { @@ -381,6 +381,6 @@ void monSendReport(SMonInfo *pMonitor) { if (pCont != NULL) { EHttpCompFlag flag = tsMonitor.comp ? HTTP_GZIP : HTTP_FLAT; taosSendHttpReport(tsMonitor.server, tsMonitor.port, pCont, strlen(pCont), flag); - free(pCont); + taosMemoryFree(pCont); } } diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 2fa7a8d663756f10b67e319ecb6640d65e3050e6..078f3ab3e61fe5699338e8662623b2008ebe76d7 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -141,12 +141,12 @@ static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: - pDst->datum.p = malloc(pSrc->node.resType.bytes + VARSTR_HEADER_SIZE); + pDst->datum.p = taosMemoryMalloc(pSrc->node.resType.bytes + VARSTR_HEADER_SIZE + 1); if (NULL == pDst->datum.p) { nodesDestroyNode(pDst); return NULL; } - memcpy(pDst->datum.p, pSrc->datum.p, pSrc->node.resType.bytes + VARSTR_HEADER_SIZE); + memcpy(pDst->datum.p, pSrc->datum.p, pSrc->node.resType.bytes + VARSTR_HEADER_SIZE + 1); break; case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_DECIMAL: @@ -210,7 +210,7 @@ static SNode* logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) { static STableMeta* tableMetaClone(const STableMeta* pSrc) { int32_t len = TABLE_META_SIZE(pSrc); - STableMeta* pDst = malloc(len); + STableMeta* pDst = taosMemoryMalloc(len); if (NULL == pDst) { return NULL; } @@ -220,7 +220,7 @@ static STableMeta* tableMetaClone(const STableMeta* pSrc) { static SVgroupsInfo* vgroupsInfoClone(const SVgroupsInfo* pSrc) { int32_t len = VGROUPS_INFO_SIZE(pSrc); - SVgroupsInfo* pDst = malloc(len); + SVgroupsInfo* pDst = taosMemoryMalloc(len); if (NULL == pDst) { return NULL; } @@ -272,11 +272,14 @@ static SNode* logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* pD COPY_SCALAR_FIELD(interval); COPY_SCALAR_FIELD(offset); COPY_SCALAR_FIELD(sliding); + COPY_SCALAR_FIELD(intervalUnit); + COPY_SCALAR_FIELD(slidingUnit); CLONE_NODE_FIELD(pFill); + COPY_SCALAR_FIELD(sessionGap); return (SNode*)pDst; } -static SNode* logicSubplanCopy(const SSubLogicPlan* pSrc, SSubLogicPlan* pDst) { +static SNode* logicSubplanCopy(const SLogicSubplan* pSrc, SLogicSubplan* pDst) { CLONE_NODE_FIELD(pNode); COPY_SCALAR_FIELD(subplanType); return (SNode*)pDst; @@ -358,7 +361,7 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) { case QUERY_NODE_LOGIC_PLAN_WINDOW: return logicWindowCopy((const SWindowLogicNode*)pNode, (SWindowLogicNode*)pDst); case QUERY_NODE_LOGIC_SUBPLAN: - return logicSubplanCopy((const SSubLogicPlan*)pNode, (SSubLogicPlan*)pDst); + return logicSubplanCopy((const SLogicSubplan*)pNode, (SLogicSubplan*)pDst); default: break; } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index f43557c33527d4d03ea45387c5d65962eee36b62..1acc3c51c5bc2ebc22fe5b08b5989abbd41979fb 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -70,6 +70,14 @@ const char* nodesNodeName(ENodeType type) { return "SlotDesc"; case QUERY_NODE_COLUMN_DEF: return "ColumnDef"; + case QUERY_NODE_DOWNSTREAM_SOURCE: + return "DownstreamSource"; + case QUERY_NODE_DATABASE_OPTIONS: + return "DatabaseOptions"; + case QUERY_NODE_TABLE_OPTIONS: + return "TableOptions"; + case QUERY_NODE_INDEX_OPTIONS: + return "IndexOptions"; case QUERY_NODE_SET_OPERATOR: return "SetOperator"; case QUERY_NODE_SELECT_STMT: @@ -78,16 +86,76 @@ const char* nodesNodeName(ENodeType type) { return "VnodeModifStmt"; case QUERY_NODE_CREATE_DATABASE_STMT: return "CreateDatabaseStmt"; + case QUERY_NODE_DROP_DATABASE_STMT: + return "DropDatabaseStmt"; + case QUERY_NODE_ALTER_DATABASE_STMT: + return "AlterDatabaseStmt"; case QUERY_NODE_CREATE_TABLE_STMT: return "CreateTableStmt"; + case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: + return "CreateSubtableClause"; + case QUERY_NODE_CREATE_MULTI_TABLE_STMT: + return "CreateMultiTableStmt"; + case QUERY_NODE_DROP_TABLE_CLAUSE: + return "DropTableClause"; + case QUERY_NODE_DROP_TABLE_STMT: + return "DropTableStmt"; + case QUERY_NODE_DROP_SUPER_TABLE_STMT: + return "DropSuperTableStmt"; + case QUERY_NODE_ALTER_TABLE_STMT: + return "AlterTableStmt"; + case QUERY_NODE_CREATE_USER_STMT: + return "CreateUserStmt"; + case QUERY_NODE_ALTER_USER_STMT: + return "AlterUserStmt"; + case QUERY_NODE_DROP_USER_STMT: + return "DropUserStmt"; case QUERY_NODE_USE_DATABASE_STMT: return "UseDatabaseStmt"; + case QUERY_NODE_CREATE_DNODE_STMT: + return "CreateDnodeStmt"; + case QUERY_NODE_DROP_DNODE_STMT: + return "DropDnodeStmt"; + case QUERY_NODE_ALTER_DNODE_STMT: + return "AlterDnodeStmt"; + case QUERY_NODE_CREATE_INDEX_STMT: + return "CreateIndexStmt"; + case QUERY_NODE_DROP_INDEX_STMT: + return "DropIndexStmt"; + case QUERY_NODE_CREATE_QNODE_STMT: + return "CreateQnodeStmt"; + case QUERY_NODE_DROP_QNODE_STMT: + return "DropQnodeStmt"; + case QUERY_NODE_CREATE_TOPIC_STMT: + return "CreateTopicStmt"; + case QUERY_NODE_DROP_TOPIC_STMT: + return "DropTopicStmt"; + case QUERY_NODE_ALTER_LOCAL_STMT: + return "AlterLocalStmt"; case QUERY_NODE_SHOW_DATABASES_STMT: return "ShowDatabaseStmt"; case QUERY_NODE_SHOW_TABLES_STMT: return "ShowTablesStmt"; - case QUERY_NODE_CREATE_TOPIC_STMT: - return "CreateTopicStmt"; + case QUERY_NODE_SHOW_STABLES_STMT: + return "ShowStablesStmt"; + case QUERY_NODE_SHOW_USERS_STMT: + return "ShowUsersStmt"; + case QUERY_NODE_SHOW_DNODES_STMT: + return "ShowDnodesStmt"; + case QUERY_NODE_SHOW_VGROUPS_STMT: + return "ShowVgroupsStmt"; + case QUERY_NODE_SHOW_MNODES_STMT: + return "ShowMnodesStmt"; + case QUERY_NODE_SHOW_MODULES_STMT: + return "ShowModulesStmt"; + case QUERY_NODE_SHOW_QNODES_STMT: + return "ShowQnodesStmt"; + case QUERY_NODE_SHOW_FUNCTIONS_STMT: + return "ShowFunctionsStmt"; + case QUERY_NODE_SHOW_INDEXES_STMT: + return "ShowIndexesStmt"; + case QUERY_NODE_SHOW_STREAMS_STMT: + return "ShowStreamsStmt"; case QUERY_NODE_LOGIC_PLAN_SCAN: return "LogicScan"; case QUERY_NODE_LOGIC_PLAN_JOIN: @@ -98,6 +166,10 @@ const char* nodesNodeName(ENodeType type) { return "LogicProject"; case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF: return "LogicVnodeModif"; + case QUERY_NODE_LOGIC_PLAN_EXCHANGE: + return "LogicExchange"; + case QUERY_NODE_LOGIC_PLAN_WINDOW: + return "LogicWindow"; case QUERY_NODE_LOGIC_SUBPLAN: return "LogicSubplan"; case QUERY_NODE_LOGIC_PLAN: @@ -124,6 +196,8 @@ const char* nodesNodeName(ENodeType type) { return "PhysiSort"; case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: return "PhysiInterval"; + case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: + return "PhysiSessionWindow"; case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: return "PhysiDispatch"; case QUERY_NODE_PHYSICAL_PLAN_INSERT: @@ -846,8 +920,37 @@ static int32_t jsonToPhysiExchangeNode(const SJson* pJson, void* pObj) { return code; } -static const char* jkIntervalPhysiPlanExprs = "Exprs"; -static const char* jkIntervalPhysiPlanFuncs = "Funcs"; +static const char* jkWindowPhysiPlanExprs = "Exprs"; +static const char* jkWindowPhysiPlanFuncs = "Funcs"; + +static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) { + const SWinodwPhysiNode* pNode = (const SWinodwPhysiNode*)pObj; + + int32_t code = physicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkWindowPhysiPlanExprs, pNode->pExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkWindowPhysiPlanFuncs, pNode->pFuncs); + } + + return code; +} + +static int32_t jsonToPhysiWindowNode(const SJson* pJson, void* pObj) { + SWinodwPhysiNode* pNode = (SWinodwPhysiNode*)pObj; + + int32_t code = jsonToPhysicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkWindowPhysiPlanExprs, &pNode->pExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkWindowPhysiPlanFuncs, &pNode->pFuncs); + } + + return code; +} + static const char* jkIntervalPhysiPlanInterval = "Interval"; static const char* jkIntervalPhysiPlanOffset = "Offset"; static const char* jkIntervalPhysiPlanSliding = "Sliding"; @@ -858,13 +961,7 @@ static const char* jkIntervalPhysiPlanFill = "Fill"; static int32_t physiIntervalNodeToJson(const void* pObj, SJson* pJson) { const SIntervalPhysiNode* pNode = (const SIntervalPhysiNode*)pObj; - int32_t code = physicPlanNodeToJson(pObj, pJson); - if (TSDB_CODE_SUCCESS == code) { - code = nodeListToJson(pJson, jkIntervalPhysiPlanExprs, pNode->pExprs); - } - if (TSDB_CODE_SUCCESS == code) { - code = nodeListToJson(pJson, jkIntervalPhysiPlanFuncs, pNode->pFuncs); - } + int32_t code = physiWindowNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkIntervalPhysiPlanInterval, pNode->interval); } @@ -890,13 +987,7 @@ static int32_t physiIntervalNodeToJson(const void* pObj, SJson* pJson) { static int32_t jsonToPhysiIntervalNode(const SJson* pJson, void* pObj) { SIntervalPhysiNode* pNode = (SIntervalPhysiNode*)pObj; - int32_t code = jsonToPhysicPlanNode(pJson, pObj); - if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeList(pJson, jkIntervalPhysiPlanExprs, &pNode->pExprs); - } - if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeList(pJson, jkIntervalPhysiPlanFuncs, &pNode->pFuncs); - } + int32_t code = jsonToPhysiWindowNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBigIntValue(pJson, jkIntervalPhysiPlanInterval, &pNode->interval); } @@ -919,6 +1010,30 @@ static int32_t jsonToPhysiIntervalNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkSessionWindowPhysiPlanGap = "Gap"; + +static int32_t physiSessionWindowNodeToJson(const void* pObj, SJson* pJson) { + const SSessionWinodwPhysiNode* pNode = (const SSessionWinodwPhysiNode*)pObj; + + int32_t code = physiWindowNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkSessionWindowPhysiPlanGap, pNode->gap); + } + + return code; +} + +static int32_t jsonToPhysiSessionWindowNode(const SJson* pJson, void* pObj) { + SSessionWinodwPhysiNode* pNode = (SSessionWinodwPhysiNode*)pObj; + + int32_t code = jsonToPhysiWindowNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkSessionWindowPhysiPlanGap, pNode->gap); + } + + return code; +} + static const char* jkDataSinkInputDataBlockDesc = "InputDataBlockDesc"; static int32_t physicDataSinkNodeToJson(const void* pObj, SJson* pJson) { @@ -1365,7 +1480,7 @@ static int32_t jsonToDatum(const SJson* pJson, void* pObj) { case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: { - pNode->datum.p = calloc(1, pNode->node.resType.bytes); + pNode->datum.p = taosMemoryCalloc(1, pNode->node.resType.bytes + VARSTR_HEADER_SIZE + 1); if (NULL == pNode->datum.p) { code = TSDB_CODE_OUT_OF_MEMORY; break; @@ -1692,6 +1807,45 @@ static int32_t groupingSetNodeToJson(const void* pObj, SJson* pJson) { return code; } +static const char* jkIntervalWindowInterval = "Interval"; +static const char* jkIntervalWindowOffset = "Offset"; +static const char* jkIntervalWindowSliding = "Sliding"; +static const char* jkIntervalWindowFill = "Fill"; + +static int32_t intervalWindowNodeToJson(const void* pObj, SJson* pJson) { + const SIntervalWindowNode* pNode = (const SIntervalWindowNode*)pObj; + + int32_t code = tjsonAddObject(pJson, jkIntervalWindowInterval, nodeToJson, pNode->pInterval); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkIntervalWindowOffset, nodeToJson, pNode->pOffset); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkIntervalWindowSliding, nodeToJson, pNode->pSliding); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkIntervalWindowFill, nodeToJson, pNode->pFill); + } + + return code; +} + +static int32_t jsonToIntervalWindowNode(const SJson* pJson, void* pObj) { + SIntervalWindowNode* pNode = (SIntervalWindowNode*)pObj; + + int32_t code = jsonToNodeObject(pJson, jkIntervalWindowInterval, &pNode->pInterval); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkIntervalWindowOffset, &pNode->pOffset); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkIntervalWindowSliding, &pNode->pSliding); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkIntervalWindowFill, &pNode->pFill); + } + + return code; +} + static const char* jkNodeListDataType = "DataType"; static const char* jkNodeListNodeList = "NodeList"; @@ -2004,8 +2158,9 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_LIMIT: case QUERY_NODE_STATE_WINDOW: case QUERY_NODE_SESSION_WINDOW: - case QUERY_NODE_INTERVAL_WINDOW: break; + case QUERY_NODE_INTERVAL_WINDOW: + return intervalWindowNodeToJson(pObj, pJson); case QUERY_NODE_NODE_LIST: return nodeListNodeToJson(pObj, pJson); case QUERY_NODE_FILL: @@ -2066,6 +2221,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { break; case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: return physiIntervalNodeToJson(pObj, pJson); + case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: + return physiSessionWindowNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: return physiDispatchNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_INSERT: @@ -2075,7 +2232,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_PHYSICAL_PLAN: return planToJson(pObj, pJson); default: - assert(0); + // assert(0); break; } nodesWarn("specificNodeToJson unknown node = %s", nodesNodeName(nodeType(pObj))); @@ -2105,7 +2262,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { // case QUERY_NODE_LIMIT: // case QUERY_NODE_STATE_WINDOW: // case QUERY_NODE_SESSION_WINDOW: - // case QUERY_NODE_INTERVAL_WINDOW: + case QUERY_NODE_INTERVAL_WINDOW: + return jsonToIntervalWindowNode(pJson, pObj); case QUERY_NODE_NODE_LIST: return jsonToNodeListNode(pJson, pObj); // case QUERY_NODE_FILL: @@ -2149,14 +2307,16 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToPhysiAggNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: return jsonToPhysiExchangeNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: + return jsonToPhysiIntervalNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: + return jsonToPhysiSessionWindowNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: return jsonToPhysiDispatchNode(pJson, pObj); case QUERY_NODE_PHYSICAL_SUBPLAN: return jsonToSubplan(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN: return jsonToPlan(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: - return jsonToPhysiIntervalNode(pJson, pObj); default: assert(0); break; diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index e3c2dff1208b427db0df79dfe1eafd2fb7682c2b..ff71c3bd585450fe2c544290cc35a625c42efc50 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -79,9 +79,14 @@ static EDealRes walkNode(SNode* pNode, ETraversalOrder order, FNodeWalker walker case QUERY_NODE_STATE_WINDOW: res = walkNode(((SStateWindowNode*)pNode)->pCol, order, walker, pContext); break; - case QUERY_NODE_SESSION_WINDOW: - res = walkNode(((SSessionWindowNode*)pNode)->pCol, order, walker, pContext); + case QUERY_NODE_SESSION_WINDOW: { + SSessionWindowNode* pSession = (SSessionWindowNode*)pNode; + res = walkNode(pSession->pCol, order, walker, pContext); + if (DEAL_RES_ERROR != res) { + res = walkNode(pSession->pGap, order, walker, pContext); + } break; + } case QUERY_NODE_INTERVAL_WINDOW: { SIntervalWindowNode* pInterval = (SIntervalWindowNode*)pNode; res = walkNode(pInterval->pInterval, order, walker, pContext); diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 14bae19cf1bc1c0b2f9738326e2dbf2a39e12e45..5345e84cdb9635c7d961d09478164927048c1a90 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -22,7 +22,7 @@ #include "thash.h" static SNode* makeNode(ENodeType type, size_t size) { - SNode* p = calloc(1, size); + SNode* p = taosMemoryCalloc(1, size); if (NULL == p) { return NULL; } @@ -160,7 +160,7 @@ SNodeptr nodesMakeNode(ENodeType type) { case QUERY_NODE_LOGIC_PLAN_WINDOW: return makeNode(type, sizeof(SWindowLogicNode)); case QUERY_NODE_LOGIC_SUBPLAN: - return makeNode(type, sizeof(SSubLogicPlan)); + return makeNode(type, sizeof(SLogicSubplan)); case QUERY_NODE_LOGIC_PLAN: return makeNode(type, sizeof(SQueryLogicPlan)); case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: @@ -185,6 +185,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SNode)); case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: return makeNode(type, sizeof(SIntervalPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: + return makeNode(type, sizeof(SSessionWinodwPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: return makeNode(type, sizeof(SDataDispatcherNode)); case QUERY_NODE_PHYSICAL_PLAN_INSERT: @@ -205,9 +207,9 @@ static EDealRes destroyNode(SNode** pNode, void* pContext) { case QUERY_NODE_VALUE: { SValueNode* pValue = (SValueNode*)*pNode; - tfree(pValue->literal); + taosMemoryFreeClear(pValue->literal); if (IS_VAR_DATA_TYPE(pValue->node.resType.type)) { - tfree(pValue->datum.p); + taosMemoryFreeClear(pValue->datum.p); } break; @@ -220,8 +222,8 @@ static EDealRes destroyNode(SNode** pNode, void* pContext) { break; case QUERY_NODE_REAL_TABLE: { SRealTableNode* pReal = (SRealTableNode*)*pNode; - tfree(pReal->pMeta); - tfree(pReal->pVgroupList); + taosMemoryFreeClear(pReal->pMeta); + taosMemoryFreeClear(pReal->pVgroupList); break; } case QUERY_NODE_TEMP_TABLE: @@ -260,8 +262,8 @@ static EDealRes destroyNode(SNode** pNode, void* pContext) { size_t size = taosArrayGetSize(pStmt->pDataBlocks); for (size_t i = 0; i < size; ++i) { SVgDataBlocks* pVg = taosArrayGetP(pStmt->pDataBlocks, i); - tfree(pVg->pData); - tfree(pVg); + taosMemoryFreeClear(pVg->pData); + taosMemoryFreeClear(pVg); } taosArrayDestroy(pStmt->pDataBlocks); break; @@ -290,7 +292,7 @@ static EDealRes destroyNode(SNode** pNode, void* pContext) { default: break; } - tfree(*pNode); + taosMemoryFreeClear(*pNode); return DEAL_RES_CONTINUE; } @@ -302,7 +304,7 @@ void nodesDestroyNode(SNodeptr pNode) { } SNodeList* nodesMakeList() { - SNodeList* p = calloc(1, sizeof(SNodeList)); + SNodeList* p = taosMemoryCalloc(1, sizeof(SNodeList)); if (NULL == p) { return NULL; } @@ -313,7 +315,7 @@ int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode) { if (NULL == pList || NULL == pNode) { return TSDB_CODE_SUCCESS; } - SListCell* p = calloc(1, sizeof(SListCell)); + SListCell* p = taosMemoryCalloc(1, sizeof(SListCell)); if (NULL == p) { terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY; @@ -332,6 +334,7 @@ int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode) { int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode) { if (NULL == pNode) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY; } int32_t code = nodesListAppend(pList, pNode); @@ -341,6 +344,17 @@ int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode) { return code; } +int32_t nodesListMakeAppend(SNodeList** pList, SNodeptr pNode) { + if (NULL == *pList) { + *pList = nodesMakeList(); + if (NULL == *pList) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; + } + } + return nodesListAppend(*pList, pNode); +} + int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc) { if (NULL == pTarget || NULL == pSrc) { return TSDB_CODE_SUCCESS; @@ -356,7 +370,7 @@ int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc) { } pTarget->pTail = pSrc->pTail; pTarget->length += pSrc->length; - tfree(pSrc); + taosMemoryFreeClear(pSrc); return TSDB_CODE_SUCCESS; } @@ -381,7 +395,7 @@ SListCell* nodesListErase(SNodeList* pList, SListCell* pCell) { } SListCell* pNext = pCell->pNext; nodesDestroyNode(pCell->pNode); - tfree(pCell); + taosMemoryFreeClear(pCell); --(pList->length); return pNext; } @@ -405,7 +419,7 @@ void nodesDestroyList(SNodeList* pList) { while (NULL != pNext) { pNext = nodesListErase(pList, pNext); } - tfree(pList); + taosMemoryFreeClear(pList); } void nodesClearList(SNodeList* pList) { @@ -417,9 +431,9 @@ void nodesClearList(SNodeList* pList) { while (NULL != pNext) { SListCell* tmp = pNext; pNext = pNext->pNext; - tfree(tmp); + taosMemoryFreeClear(tmp); } - tfree(pList); + taosMemoryFreeClear(pList); } void* nodesGetValueFromNode(SValueNode *pNode) { diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index eb33dec7210677c2071fb1130f12e1718ba43b0d..358edcb279f0ff7f7f328e7c86b6c5186a412885 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -100,7 +100,7 @@ SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const STok SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft, SNode* pRight, SNode* pJoinCond); SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset); SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder); -SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, const SToken* pVal); +SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap); SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pCol); SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, SNode* pFill); SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues); @@ -150,9 +150,9 @@ SNode* createDropUserStmt(SAstCreateContext* pCxt, SToken* pUserName); SNode* createCreateDnodeStmt(SAstCreateContext* pCxt, const SToken* pFqdn, const SToken* pPort); SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode); SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const SToken* pConfig, const SToken* pValue); -SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, SToken* pIndexName, SToken* pTableName, SNodeList* pCols, SNode* pOptions); +SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool ignoreExists, SToken* pIndexName, SToken* pTableName, SNodeList* pCols, SNode* pOptions); SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInterval, SNode* pOffset, SNode* pSliding); -SNode* createDropIndexStmt(SAstCreateContext* pCxt, SToken* pIndexName, SToken* pTableName); +SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pIndexName, SToken* pTableName); SNode* createCreateQnodeStmt(SAstCreateContext* pCxt, const SToken* pDnodeId); SNode* createDropQnodeStmt(SAstCreateContext* pCxt, const SToken* pDnodeId); SNode* createCreateTopicStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, SNode* pQuery, const SToken* pSubscribeDbName); diff --git a/source/libs/parser/inc/parInsertData.h b/source/libs/parser/inc/parInsertData.h index 67ff2d1ae0aa8afc3fd6806312f16f2ab688adbf..acd021572d4de499f85bbee35b5366c6d68e259c 100644 --- a/source/libs/parser/inc/parInsertData.h +++ b/source/libs/parser/inc/parInsertData.h @@ -118,7 +118,7 @@ static FORCE_INLINE void getMemRowAppendInfo(SSchema *pSchema, uint8_t rowType, } static FORCE_INLINE int32_t setBlockInfo(SSubmitBlk *pBlocks, STableDataBlocks* dataBuf, int32_t numOfRows) { - pBlocks->tid = dataBuf->pTableMeta->suid; + pBlocks->suid = (TSDB_NORMAL_TABLE == dataBuf->pTableMeta->tableType ? dataBuf->pTableMeta->uid : dataBuf->pTableMeta->suid); pBlocks->uid = dataBuf->pTableMeta->uid; pBlocks->sversion = dataBuf->pTableMeta->sversion; diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 4bb555681418552cba3a05658ef007e9917eda9b..529fbb55c8e65521d8d3c20d26c963628750009f 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -313,10 +313,11 @@ func_name_list(A) ::= func_name_list(B) NK_COMMA col_name(C). func_name(A) ::= function_name(B). { A = createFunctionNode(pCxt, &B, NULL); } /************************************************ create index ********************************************************/ -cmd ::= CREATE SMA INDEX index_name(A) ON table_name(B) index_options(C). { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, &A, &B, NULL, C); } -cmd ::= CREATE FULLTEXT INDEX - index_name(A) ON table_name(B) NK_LP col_name_list(C) NK_RP. { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, &A, &B, C, NULL); } -cmd ::= DROP INDEX index_name(A) ON table_name(B). { pCxt->pRootNode = createDropIndexStmt(pCxt, &A, &B); } +cmd ::= CREATE SMA INDEX not_exists_opt(D) + index_name(A) ON table_name(B) index_options(C). { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, D, &A, &B, NULL, C); } +cmd ::= CREATE FULLTEXT INDEX not_exists_opt(D) + index_name(A) ON table_name(B) NK_LP col_name_list(C) NK_RP. { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, D, &A, &B, C, NULL); } +cmd ::= DROP INDEX exists_opt(C) index_name(A) ON table_name(B). { pCxt->pRootNode = createDropIndexStmt(pCxt, C, &A, &B); } index_options(A) ::= . { A = NULL; } index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL @@ -624,7 +625,7 @@ partition_by_clause_opt(A) ::= PARTITION BY expression_list(B). twindow_clause_opt(A) ::= . { A = NULL; } twindow_clause_opt(A) ::= - SESSION NK_LP column_reference(B) NK_COMMA NK_INTEGER(C) NK_RP. { A = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, B), &C); } + SESSION NK_LP column_reference(B) NK_COMMA duration_literal(C) NK_RP. { A = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C)); } twindow_clause_opt(A) ::= STATE_WINDOW NK_LP column_reference(B) NK_RP. { A = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, B)); } twindow_clause_opt(A) ::= INTERVAL NK_LP duration_literal(B) NK_RP sliding_opt(C) fill_opt(D). { A = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, B), NULL, C, D); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index fbf74b96a3b6fe3f1485fb3ce2887db69a8856df..9630b0d68ce509248a61f01f6d7364b718e3279f 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -481,7 +481,7 @@ SNode* createRawExprNodeExt(SAstCreateContext* pCxt, const SToken* pStart, const SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { CHECK_RAW_EXPR_NODE(pNode); SNode* tmp = ((SRawExprNode*)pNode)->pNode; - tfree(pNode); + taosMemoryFreeClear(pNode); return tmp; } @@ -679,11 +679,11 @@ SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order return (SNode*)orderByExpr; } -SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, const SToken* pVal) { +SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap) { SSessionWindowNode* session = (SSessionWindowNode*)nodesMakeNode(QUERY_NODE_SESSION_WINDOW); CHECK_OUT_OF_MEM(session); session->pCol = pCol; - // session->gap = getInteger(pVal); + session->pGap = pGap; return (SNode*)session; } @@ -1160,13 +1160,14 @@ SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const return (SNode*)pStmt; } -SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, SToken* pIndexName, SToken* pTableName, SNodeList* pCols, SNode* pOptions) { +SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool ignoreExists, SToken* pIndexName, SToken* pTableName, SNodeList* pCols, SNode* pOptions) { if (!checkIndexName(pCxt, pIndexName) || !checkTableName(pCxt, pTableName)) { return NULL; } SCreateIndexStmt* pStmt = nodesMakeNode(QUERY_NODE_CREATE_INDEX_STMT); CHECK_OUT_OF_MEM(pStmt); pStmt->indexType = type; + pStmt->ignoreExists = ignoreExists; strncpy(pStmt->indexName, pIndexName->z, pIndexName->n); strncpy(pStmt->tableName, pTableName->z, pTableName->n); pStmt->pCols = pCols; @@ -1184,12 +1185,13 @@ SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInt return (SNode*)pOptions; } -SNode* createDropIndexStmt(SAstCreateContext* pCxt, SToken* pIndexName, SToken* pTableName) { +SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pIndexName, SToken* pTableName) { if (!checkIndexName(pCxt, pIndexName) || !checkTableName(pCxt, pTableName)) { return NULL; } SDropIndexStmt* pStmt = nodesMakeNode(QUERY_NODE_DROP_INDEX_STMT); CHECK_OUT_OF_MEM(pStmt); + pStmt->ignoreNotExists = ignoreNotExists; strncpy(pStmt->indexName, pIndexName->z, pIndexName->n); strncpy(pStmt->tableName, pTableName->z, pTableName->n); return (SNode*)pStmt; diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index e992f150aae875eb2620dcd5f40c347054f9f820..5c38ccaff80cfa553c5d6ed75ed3ebb6400483e6 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -29,7 +29,7 @@ extern void ParseTrace(FILE*, char*); int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery) { SAstCreateContext cxt; initAstCreateContext(pParseCxt, &cxt); - void *pParser = ParseAlloc(malloc); + void *pParser = ParseAlloc((FMalloc)taosMemoryMalloc); int32_t i = 0; while (1) { SToken t0 = {0}; @@ -73,9 +73,9 @@ int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery) { } abort_parse: - ParseFree(pParser, free); + ParseFree(pParser, (FFree)taosMemoryFree); if (cxt.valid) { - *pQuery = calloc(1, sizeof(SQuery)); + *pQuery = taosMemoryCalloc(1, sizeof(SQuery)); if (NULL == *pQuery) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 213a3956c17bc760719af8844c4103b8e63a41c5..b2fc39d0645bbf30839a08eae24b20f62c3307ff 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -48,11 +48,6 @@ } \ } while (0) -enum { - TSDB_USE_SERVER_TS = 0, - TSDB_USE_CLI_TS = 1, -}; - typedef struct SInsertParseContext { SParseContext* pComCxt; // input char *pSql; // input @@ -264,7 +259,7 @@ static void buildMsgHeader(STableDataBlocks* src, SVgDataBlocks* blocks) { while (numOfBlocks--) { int32_t dataLen = blk->dataLen; blk->uid = htobe64(blk->uid); - blk->tid = htonl(blk->tid); + blk->suid = htobe64(blk->suid); blk->padding = htonl(blk->padding); blk->sversion = htonl(blk->sversion); blk->dataLen = htonl(blk->dataLen); @@ -282,7 +277,7 @@ static int32_t buildOutput(SInsertParseContext* pCxt) { } for (size_t i = 0; i < numOfVg; ++i) { STableDataBlocks* src = taosArrayGetP(pCxt->pVgDataBlocks, i); - SVgDataBlocks* dst = calloc(1, sizeof(SVgDataBlocks)); + SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == dst) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -303,20 +298,7 @@ static int32_t checkTimestamp(STableDataBlocks *pDataBlocks, const char *start) } TSKEY k = *(TSKEY *)start; - - if (k == INT64_MIN) { - if (pDataBlocks->tsSource == TSDB_USE_CLI_TS) { - return TSDB_CODE_FAILED; // client time/server time can not be mixed - } - pDataBlocks->tsSource = TSDB_USE_SERVER_TS; - } else { - if (pDataBlocks->tsSource == TSDB_USE_SERVER_TS) { - return TSDB_CODE_FAILED; // client time/server time can not be mixed - } - pDataBlocks->tsSource = TSDB_USE_CLI_TS; - } - - if (k <= pDataBlocks->prevTS && (pDataBlocks->tsSource == TSDB_USE_CLI_TS)) { + if (k <= pDataBlocks->prevTS) { pDataBlocks->ordered = false; } @@ -702,7 +684,7 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* pColList->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; if (!isOrdered) { - pColList->colIdxInfo = calloc(pColList->numOfBound, sizeof(SBoundIdxInfo)); + pColList->colIdxInfo = taosMemoryCalloc(pColList->numOfBound, sizeof(SBoundIdxInfo)); if (NULL == pColList->colIdxInfo) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -778,7 +760,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pTagsSchema, // todo construct payload - tfree(row); + taosMemoryFreeClear(row); return 0; } @@ -921,7 +903,7 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* da } static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) { - tfree(pCxt->pTableMeta); + taosMemoryFreeClear(pCxt->pTableMeta); destroyBoundColumnInfo(&pCxt->tags); tdDestroyKVRowBuilder(&pCxt->tagsBuilder); } @@ -931,16 +913,16 @@ static void destroyDataBlock(STableDataBlocks* pDataBlock) { return; } - tfree(pDataBlock->pData); + taosMemoryFreeClear(pDataBlock->pData); if (!pDataBlock->cloned) { // free the refcount for metermeta if (pDataBlock->pTableMeta != NULL) { - tfree(pDataBlock->pTableMeta); + taosMemoryFreeClear(pDataBlock->pTableMeta); } destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); } - tfree(pDataBlock); + taosMemoryFreeClear(pDataBlock); } static void destroyInsertParseContext(SInsertParseContext* pCxt) { @@ -1047,7 +1029,7 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - *pQuery = calloc(1, sizeof(SQuery)); + *pQuery = taosMemoryCalloc(1, sizeof(SQuery)); if (NULL == *pQuery) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index da5a652018f7066b7f176462e5ad633cfb834bd1..e516053b1e9b62c5b8ce62b09fd157b818c718c6 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -47,8 +47,8 @@ void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, int32_t pColList->numOfCols = numOfCols; pColList->numOfBound = numOfCols; pColList->orderStatus = ORDER_STATUS_ORDERED; // default is ORDERED for non-bound mode - pColList->boundedColumns = calloc(pColList->numOfCols, sizeof(int32_t)); - pColList->cols = calloc(pColList->numOfCols, sizeof(SBoundColumn)); + pColList->boundedColumns = taosMemoryCalloc(pColList->numOfCols, sizeof(int32_t)); + pColList->cols = taosMemoryCalloc(pColList->numOfCols, sizeof(SBoundColumn)); pColList->colIdxInfo = NULL; pColList->flen = 0; pColList->allNullLen = 0; @@ -103,14 +103,14 @@ int32_t boundIdxCompar(const void *lhs, const void *rhs) { } void destroyBoundColumnInfo(SParsedDataColInfo* pColList) { - tfree(pColList->boundedColumns); - tfree(pColList->cols); - tfree(pColList->colIdxInfo); + taosMemoryFreeClear(pColList->boundedColumns); + taosMemoryFreeClear(pColList->cols); + taosMemoryFreeClear(pColList->colIdxInfo); } static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, const STableMeta* pTableMeta, STableDataBlocks** dataBlocks) { - STableDataBlocks* dataBuf = (STableDataBlocks*)calloc(1, sizeof(STableDataBlocks)); + STableDataBlocks* dataBuf = (STableDataBlocks*)taosMemoryCalloc(1, sizeof(STableDataBlocks)); if (dataBuf == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -123,9 +123,9 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star dataBuf->nAllocSize = dataBuf->headerSize * 2; } - dataBuf->pData = malloc(dataBuf->nAllocSize); + dataBuf->pData = taosMemoryMalloc(dataBuf->nAllocSize); if (dataBuf->pData == NULL) { - tfree(dataBuf); + taosMemoryFreeClear(dataBuf); return TSDB_CODE_TSC_OUT_OF_MEMORY; } memset(dataBuf->pData, 0, sizeof(SSubmitBlk)); @@ -141,7 +141,6 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star dataBuf->prevTS = INT64_MIN; dataBuf->rowSize = rowSize; dataBuf->size = startOffset; - dataBuf->tsSource = -1; dataBuf->vgId = dataBuf->pTableMeta->vgId; assert(defaultSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); @@ -191,16 +190,16 @@ static void destroyDataBlock(STableDataBlocks* pDataBlock) { return; } - tfree(pDataBlock->pData); + taosMemoryFreeClear(pDataBlock->pData); if (!pDataBlock->cloned) { // free the refcount for metermeta if (pDataBlock->pTableMeta != NULL) { - tfree(pDataBlock->pTableMeta); + taosMemoryFreeClear(pDataBlock->pTableMeta); } destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); } - tfree(pDataBlock); + taosMemoryFreeClear(pDataBlock); } void destroyBlockArrayList(SArray* pDataBlockList) { @@ -284,7 +283,7 @@ int sortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlkKey // allocate memory size_t nAlloc = nRows * sizeof(SBlockKeyTuple); if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { - char *tmp = realloc(pBlkKeyInfo->pKeyTuple, nAlloc); + char *tmp = taosMemoryRealloc(pBlkKeyInfo->pKeyTuple, nAlloc); if (tmp == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -434,7 +433,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t if (ret != TSDB_CODE_SUCCESS) { taosHashCleanup(pVnodeDataBlockHashList); destroyBlockArrayList(pVnodeDataBlockList); - tfree(blkKeyInfo.pKeyTuple); + taosMemoryFreeClear(blkKeyInfo.pKeyTuple); return ret; } @@ -445,14 +444,14 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t if (dataBuf->nAllocSize < destSize) { dataBuf->nAllocSize = (uint32_t)(destSize * 1.5); - char* tmp = realloc(dataBuf->pData, dataBuf->nAllocSize); + char* tmp = taosMemoryRealloc(dataBuf->pData, dataBuf->nAllocSize); if (tmp != NULL) { dataBuf->pData = tmp; } else { // failed to allocate memory, free already allocated memory and return error code taosHashCleanup(pVnodeDataBlockHashList); destroyBlockArrayList(pVnodeDataBlockList); - tfree(dataBuf->pData); - tfree(blkKeyInfo.pKeyTuple); + taosMemoryFreeClear(dataBuf->pData); + taosMemoryFreeClear(blkKeyInfo.pKeyTuple); return TSDB_CODE_TSC_OUT_OF_MEMORY; } } @@ -463,8 +462,8 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t if ((code = sortRemoveDataBlockDupRows(pOneTableBlock, &blkKeyInfo)) != 0) { taosHashCleanup(pVnodeDataBlockHashList); destroyBlockArrayList(pVnodeDataBlockList); - tfree(dataBuf->pData); - tfree(blkKeyInfo.pKeyTuple); + taosMemoryFreeClear(dataBuf->pData); + taosMemoryFreeClear(blkKeyInfo.pKeyTuple); return code; } ASSERT(blkKeyInfo.pKeyTuple != NULL && pBlocks->numOfRows > 0); @@ -493,7 +492,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t // free the table data blocks; taosHashCleanup(pVnodeDataBlockHashList); - tfree(blkKeyInfo.pKeyTuple); + taosMemoryFreeClear(blkKeyInfo.pKeyTuple); *pVgDataBlocks = pVnodeDataBlockList; return TSDB_CODE_SUCCESS; } @@ -510,7 +509,7 @@ int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t remain = pDataBlock->nAllocSize - pDataBlock->size; } - char *tmp = realloc(pDataBlock->pData, (size_t)pDataBlock->nAllocSize); + char *tmp = taosMemoryRealloc(pDataBlock->pData, (size_t)pDataBlock->nAllocSize); if (tmp != NULL) { pDataBlock->pData = tmp; memset(pDataBlock->pData + pDataBlock->size, 0, pDataBlock->nAllocSize - pDataBlock->size); diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 1dc1945f23a4934bae6b2ea1d7a726c1fcbd365e..79651cd325fe6c9582ab875a6bc713689c8699cd 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -594,7 +594,7 @@ SToken tscReplaceStrToken(char **str, SToken *token, const char* newToken) { int32_t bsize = (int32_t)((uint64_t)token->z - (uint64_t)src); SToken ntoken; - *str = calloc(1, size); + *str = taosMemoryCalloc(1, size); strncpy(*str, src, bsize); strcat(*str, newToken); @@ -603,7 +603,7 @@ SToken tscReplaceStrToken(char **str, SToken *token, const char* newToken) { ntoken.n = (uint32_t)nsize; ntoken.z = *str + bsize; - tfree(src); + taosMemoryFreeClear(src); return ntoken; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 100fc3f1075fbf25de9833698e07860ec49a2535..2d1882245d5b8e5cef4e883a19910b3bdf6c66e2 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -383,7 +383,7 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: { - pVal->datum.p = calloc(1, pVal->node.resType.bytes + VARSTR_HEADER_SIZE); + pVal->datum.p = taosMemoryCalloc(1, pVal->node.resType.bytes + VARSTR_HEADER_SIZE + 1); if (NULL == pVal->datum.p) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY); } @@ -585,7 +585,7 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) static int32_t toVgroupsInfo(SArray* pVgs, SVgroupsInfo** pVgsInfo) { size_t vgroupNum = taosArrayGetSize(pVgs); - *pVgsInfo = calloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo) * vgroupNum); + *pVgsInfo = taosMemoryCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo) * vgroupNum); if (NULL == *pVgsInfo) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -640,7 +640,7 @@ static int32_t setTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTa } else if (TSDB_SYSTEM_TABLE == pRealTable->pMeta->tableType) { code = setSysTableVgroupList(pCxt, pName, pRealTable); } else { - pRealTable->pVgroupList = calloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo)); + pRealTable->pVgroupList = taosMemoryCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo)); if (NULL == pRealTable->pVgroupList) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -824,11 +824,30 @@ static int32_t translateGroupBy(STranslateContext* pCxt, SNodeList* pGroupByList return translateExprList(pCxt, pGroupByList); } +static int32_t translateIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* pInterval) { + SValueNode* pIntervalVal = (SValueNode*)pInterval->pInterval; + SValueNode* pIntervalOffset = (SValueNode*)pInterval->pOffset; + SValueNode* pSliding = (SValueNode*)pInterval->pSliding; + if (pIntervalVal->datum.i <= 0) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTERVAL_VALUE_TOO_SMALL, pIntervalVal->literal); + } + return TSDB_CODE_SUCCESS; +} + static int32_t doTranslateWindow(STranslateContext* pCxt, SNode* pWindow) { + switch (nodeType(pWindow)) { + case QUERY_NODE_INTERVAL_WINDOW: + return translateIntervalWindow(pCxt, (SIntervalWindowNode*)pWindow); + default: + break; + } return TSDB_CODE_SUCCESS; } static int32_t translateWindow(STranslateContext* pCxt, SNode* pWindow) { + if (NULL == pWindow) { + return TSDB_CODE_SUCCESS; + } pCxt->currClause = SQL_CLAUSE_WINDOW; int32_t code = translateExpr(pCxt, pWindow); if (TSDB_CODE_SUCCESS == code) { @@ -913,14 +932,14 @@ static int32_t translateCreateDatabase(STranslateContext* pCxt, SCreateDatabaseS SCreateDbReq createReq = {0}; buildCreateDbReq(pCxt, pStmt, &createReq); - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_DB; pCxt->pCmdMsg->msgLen = tSerializeSCreateDbReq(NULL, 0, &createReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -936,14 +955,14 @@ static int32_t translateDropDatabase(STranslateContext* pCxt, SDropDatabaseStmt* tNameGetFullDbName(&name, dropReq.db); dropReq.ignoreNotExists = pStmt->ignoreNotExists; - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_DROP_DB; pCxt->pCmdMsg->msgLen = tSerializeSDropDbReq(NULL, 0, &dropReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -971,14 +990,14 @@ static int32_t translateAlterDatabase(STranslateContext* pCxt, SAlterDatabaseStm SAlterDbReq alterReq = {0}; buildAlterDbReq(pCxt, pStmt, &alterReq); - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_ALTER_DB; pCxt->pCmdMsg->msgLen = tSerializeSAlterDbReq(NULL, 0, &alterReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1022,7 +1041,7 @@ static int32_t translateCreateSuperTable(STranslateContext* pCxt, SCreateTableSt strcpy(tableName.tname, pStmt->tableName); tNameExtractFullName(&tableName, createReq.name); - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { tFreeSMCreateStbReq(&createReq); return TSDB_CODE_OUT_OF_MEMORY; @@ -1030,7 +1049,7 @@ static int32_t translateCreateSuperTable(STranslateContext* pCxt, SCreateTableSt pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_STB; pCxt->pCmdMsg->msgLen = tSerializeSMCreateStbReq(NULL, 0, &createReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { tFreeSMCreateStbReq(&createReq); return TSDB_CODE_OUT_OF_MEMORY; @@ -1046,14 +1065,14 @@ static int32_t doTranslateDropSuperTable(STranslateContext* pCxt, const SName* p tNameExtractFullName(pTableName, dropReq.name); dropReq.igNotExists = ignoreNotExists; - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_DROP_STB; pCxt->pCmdMsg->msgLen = tSerializeSMDropStbReq(NULL, 0, &dropReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1076,7 +1095,7 @@ static int32_t translateDropTable(STranslateContext* pCxt, SDropTableStmt* pStmt // todo : drop normal table or child table code = TSDB_CODE_FAILED; } - tfree(pTableMeta); + taosMemoryFreeClear(pTableMeta); } return code; @@ -1140,14 +1159,14 @@ static int32_t translateAlterTable(STranslateContext* pCxt, SAlterTableStmt* pSt } } - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_ALTER_STB; pCxt->pCmdMsg->msgLen = tSerializeSMAlterStbReq(NULL, 0, &alterReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1166,14 +1185,14 @@ static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* p return code; } - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_USE_DB; pCxt->pCmdMsg->msgLen = tSerializeSUseDbReq(NULL, 0, &usedbReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1189,14 +1208,14 @@ static int32_t translateCreateUser(STranslateContext* pCxt, SCreateUserStmt* pSt createReq.superUser = 0; strcpy(createReq.pass, pStmt->password); - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_USER; pCxt->pCmdMsg->msgLen = tSerializeSCreateUserReq(NULL, 0, &createReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1215,14 +1234,14 @@ static int32_t translateAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt strcpy(alterReq.dbname, pCxt->pParseCxt->db); } - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_ALTER_USER; pCxt->pCmdMsg->msgLen = tSerializeSAlterUserReq(NULL, 0, &alterReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1235,14 +1254,14 @@ static int32_t translateDropUser(STranslateContext* pCxt, SDropUserStmt* pStmt) SDropUserReq dropReq = {0}; strcpy(dropReq.user, pStmt->useName); - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_DROP_USER; pCxt->pCmdMsg->msgLen = tSerializeSDropUserReq(NULL, 0, &dropReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1256,14 +1275,14 @@ static int32_t translateCreateDnode(STranslateContext* pCxt, SCreateDnodeStmt* p strcpy(createReq.fqdn, pStmt->fqdn); createReq.port = pStmt->port; - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_DNODE; pCxt->pCmdMsg->msgLen = tSerializeSCreateDnodeReq(NULL, 0, &createReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1278,14 +1297,14 @@ static int32_t translateDropDnode(STranslateContext* pCxt, SDropDnodeStmt* pStmt strcpy(dropReq.fqdn, pStmt->fqdn); dropReq.port = pStmt->port; - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_DROP_DNODE; pCxt->pCmdMsg->msgLen = tSerializeSDropDnodeReq(NULL, 0, &dropReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1300,14 +1319,14 @@ static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pSt strcpy(cfgReq.config, pStmt->config); strcpy(cfgReq.value, pStmt->value); - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_CONFIG_DNODE; pCxt->pCmdMsg->msgLen = tSerializeSMCfgDnodeReq(NULL, 0, &cfgReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1347,14 +1366,14 @@ static int32_t translateShow(STranslateContext* pCxt, SShowStmt* pStmt) { // tNameGetFullDbName(&name, showReq.db); // } - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_SHOW; pCxt->pCmdMsg->msgLen = tSerializeSShowReq(NULL, 0, &showReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1364,7 +1383,7 @@ static int32_t translateShow(STranslateContext* pCxt, SShowStmt* pStmt) { } static int32_t translateShowTables(STranslateContext* pCxt) { - SVShowTablesReq* pShowReq = calloc(1, sizeof(SVShowTablesReq)); + SVShowTablesReq* pShowReq = taosMemoryCalloc(1, sizeof(SVShowTablesReq)); SArray* array = NULL; int32_t code = getDBVgInfo(pCxt, pCxt->pParseCxt->db, &array); @@ -1374,7 +1393,7 @@ static int32_t translateShowTables(STranslateContext* pCxt) { SVgroupInfo* info = taosArrayGet(array, 0); pShowReq->head.vgId = htonl(info->vgId); - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1387,52 +1406,127 @@ static int32_t translateShowTables(STranslateContext* pCxt) { return TSDB_CODE_SUCCESS; } -static int32_t translateCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { - SVCreateTSmaReq createSmaReq = {0}; +static int32_t getSmaIndexDstVgId(STranslateContext* pCxt, char* pTableName, int32_t* pVgId) { + SVgroupInfo vg = {0}; + int32_t code = getTableHashVgroup(pCxt, pCxt->pParseCxt->db, pTableName, &vg); + if (TSDB_CODE_SUCCESS == code) { + *pVgId = vg.vgId; + } + return code; +} - if (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pInterval) || - (NULL != pStmt->pOptions->pOffset && DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pOffset)) || - (NULL != pStmt->pOptions->pSliding && DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pSliding))) { - return pCxt->errCode; +static int32_t getSmaIndexSql(STranslateContext* pCxt, char** pSql, int32_t* pLen) { + *pSql = strdup(pCxt->pParseCxt->pSql); + if (NULL == *pSql) { + return TSDB_CODE_OUT_OF_MEMORY; + } + *pLen = pCxt->pParseCxt->sqlLen + 1; + return TSDB_CODE_SUCCESS; +} + +static int32_t getSmaIndexExpr(STranslateContext* pCxt, SCreateIndexStmt* pStmt, char** pExpr, int32_t* pLen) { + return nodesListToString(pStmt->pOptions->pFuncs, false, pExpr, pLen); +} + +static int32_t getSmaIndexBuildAst(STranslateContext* pCxt, SCreateIndexStmt* pStmt, char** pAst, int32_t* pLen) { + SSelectStmt* pSelect = nodesMakeNode(QUERY_NODE_SELECT_STMT); + if (NULL == pSelect) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SRealTableNode* pTable = nodesMakeNode(QUERY_NODE_REAL_TABLE); + if (NULL == pTable) { + nodesDestroyNode(pSelect); + return TSDB_CODE_OUT_OF_MEMORY; } + strcpy(pTable->table.dbName, pCxt->pParseCxt->db); + strcpy(pTable->table.tableName, pStmt->tableName); + pSelect->pFromTable = (SNode*)pTable; - createSmaReq.tSma.intervalUnit = ((SValueNode*)pStmt->pOptions->pInterval)->unit; - createSmaReq.tSma.slidingUnit = (NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->unit : 0); - strcpy(createSmaReq.tSma.indexName, pStmt->indexName); + pSelect->pProjectionList = nodesCloneList(pStmt->pOptions->pFuncs); + if (NULL == pTable) { + nodesDestroyNode(pSelect); + return TSDB_CODE_OUT_OF_MEMORY; + } - SName name; - name.type = TSDB_TABLE_NAME_T; - name.acctId = pCxt->pParseCxt->acctId; + SIntervalWindowNode* pInterval = nodesMakeNode(QUERY_NODE_INTERVAL_WINDOW); + if (NULL == pInterval) { + nodesDestroyNode(pSelect); + return TSDB_CODE_OUT_OF_MEMORY; + } + pSelect->pWindow = (SNode*)pInterval; + pInterval->pInterval = nodesCloneNode(pStmt->pOptions->pInterval); + pInterval->pOffset = nodesCloneNode(pStmt->pOptions->pOffset); + pInterval->pSliding = nodesCloneNode(pStmt->pOptions->pSliding); + if (NULL == pInterval->pInterval || (NULL != pStmt->pOptions->pOffset && NULL == pInterval->pOffset) || + (NULL != pStmt->pOptions->pSliding && NULL == pInterval->pSliding)) { + nodesDestroyNode(pSelect); + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = translateQuery(pCxt, (SNode*)pSelect); + if (TSDB_CODE_SUCCESS == code) { + code = nodesNodeToString(pSelect, false, pAst, pLen); + } + nodesDestroyNode(pSelect); + return code; +} + +static int32_t buildCreateSmaReq(STranslateContext* pCxt, SCreateIndexStmt* pStmt, SMCreateSmaReq* pReq) { + SName name = { .type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId }; strcpy(name.dbname, pCxt->pParseCxt->db); + strcpy(name.tname, pStmt->indexName); + tNameExtractFullName(&name, pReq->name); strcpy(name.tname, pStmt->tableName); - STableMeta* pMeta = NULL; - int32_t code = catalogGetTableMeta(pCxt->pParseCxt->pCatalog, pCxt->pParseCxt->pTransporter, &pCxt->pParseCxt->mgmtEpSet, &name, &pMeta); - if (TSDB_CODE_SUCCESS != code) { - return code; + name.tname[strlen(pStmt->tableName)] = '\0'; + tNameExtractFullName(&name, pReq->stb); + pReq->igExists = pStmt->ignoreExists; + pReq->interval = ((SValueNode*)pStmt->pOptions->pInterval)->datum.i; + pReq->intervalUnit = ((SValueNode*)pStmt->pOptions->pInterval)->unit; + pReq->offset = (NULL != pStmt->pOptions->pOffset ? ((SValueNode*)pStmt->pOptions->pOffset)->datum.i : 0); + pReq->sliding = (NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->datum.i : pReq->interval); + pReq->slidingUnit = (NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->unit : pReq->intervalUnit); + + int32_t code = getSmaIndexDstVgId(pCxt, pStmt->tableName, &pReq->dstVgId); + if (TSDB_CODE_SUCCESS == code) { + code = getSmaIndexSql(pCxt, &pReq->sql, &pReq->sqlLen); + } + if (TSDB_CODE_SUCCESS == code) { + code = getSmaIndexExpr(pCxt, pStmt, &pReq->expr, &pReq->exprLen); + } + if (TSDB_CODE_SUCCESS == code) { + code = getSmaIndexBuildAst(pCxt, pStmt, &pReq->ast, &pReq->astLen); } - createSmaReq.tSma.tableUid = pMeta->uid; - createSmaReq.tSma.interval = ((SValueNode*)pStmt->pOptions->pInterval)->datum.i; - createSmaReq.tSma.sliding = (NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->datum.i : 0); - code = nodesListToString(pStmt->pOptions->pFuncs, false, &createSmaReq.tSma.expr, &createSmaReq.tSma.exprLen); + return code; +} + +static int32_t translateCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { + if (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pInterval) || + (NULL != pStmt->pOptions->pOffset && DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pOffset)) || + (NULL != pStmt->pOptions->pSliding && DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pSliding))) { + return pCxt->errCode; + } + + SMCreateSmaReq createSmaReq = {0}; + int32_t code = buildCreateSmaReq(pCxt, pStmt, &createSmaReq); if (TSDB_CODE_SUCCESS != code) { return code; } - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; - pCxt->pCmdMsg->msgType = TDMT_VND_CREATE_SMA; - pCxt->pCmdMsg->msgLen = tSerializeSVCreateTSmaReq(NULL, &createSmaReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_SMA; + pCxt->pCmdMsg->msgLen = tSerializeSMCreateSmaReq(NULL, 0, &createSmaReq); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } - void* pBuf = pCxt->pCmdMsg->pMsg; - tSerializeSVCreateTSmaReq(&pBuf, &createSmaReq); - tdDestroyTSma(&createSmaReq.tSma); + tSerializeSMCreateSmaReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &createSmaReq); + tFreeSMCreateSmaReq(&createSmaReq); return TSDB_CODE_SUCCESS; } @@ -1450,14 +1544,14 @@ static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt SVDropTSmaReq dropSmaReq = {0}; strcpy(dropSmaReq.indexName, pStmt->indexName); - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_VND_DROP_SMA; pCxt->pCmdMsg->msgLen = tSerializeSVDropTSmaReq(NULL, &dropSmaReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1470,14 +1564,14 @@ static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt static int32_t translateCreateQnode(STranslateContext* pCxt, SCreateQnodeStmt* pStmt) { SMCreateQnodeReq createReq = { .dnodeId = pStmt->dnodeId }; - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_DND_CREATE_QNODE; pCxt->pCmdMsg->msgLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1489,14 +1583,14 @@ static int32_t translateCreateQnode(STranslateContext* pCxt, SCreateQnodeStmt* p static int32_t translateDropQnode(STranslateContext* pCxt, SDropQnodeStmt* pStmt) { SDDropQnodeReq dropReq = { .dnodeId = pStmt->dnodeId }; - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_DND_DROP_QNODE; pCxt->pCmdMsg->msgLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1532,14 +1626,14 @@ static int32_t translateCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* p tNameExtractFullName(&name, createReq.name); createReq.igExists = pStmt->ignoreExists; - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_TOPIC; pCxt->pCmdMsg->msgLen = tSerializeSCMCreateTopicReq(NULL, 0, &createReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1558,14 +1652,14 @@ static int32_t translateDropTopic(STranslateContext* pCxt, SDropTopicStmt* pStmt tNameExtractFullName(&name, dropReq.name); dropReq.igNotExists = pStmt->ignoreNotExists; - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_DROP_TOPIC; pCxt->pCmdMsg->msgLen = tSerializeSMDropTopicReq(NULL, 0, &dropReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1677,24 +1771,27 @@ static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode) { return code; } -static int32_t setReslutSchema(STranslateContext* pCxt, SQuery* pQuery) { - if (QUERY_NODE_SELECT_STMT == nodeType(pQuery->pRoot)) { - SSelectStmt* pSelect = (SSelectStmt*)pQuery->pRoot; - pQuery->numOfResCols = LIST_LENGTH(pSelect->pProjectionList); - pQuery->pResSchema = calloc(pQuery->numOfResCols, sizeof(SSchema)); - if (NULL == pQuery->pResSchema) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_OUT_OF_MEMORY); +int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema) { + if (QUERY_NODE_SELECT_STMT == nodeType(pRoot)) { + SSelectStmt* pSelect = (SSelectStmt*) pRoot; + *numOfCols = LIST_LENGTH(pSelect->pProjectionList); + *pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema)); + if (NULL == (*pSchema)) { + return TSDB_CODE_OUT_OF_MEMORY; } + SNode* pNode; int32_t index = 0; FOREACH(pNode, pSelect->pProjectionList) { SExprNode* pExpr = (SExprNode*)pNode; - pQuery->pResSchema[index].type = pExpr->resType.type; - pQuery->pResSchema[index].bytes = pExpr->resType.bytes; - strcpy(pQuery->pResSchema[index].name, pExpr->aliasName); + (*pSchema)[index].type = pExpr->resType.type; + (*pSchema)[index].bytes = pExpr->resType.bytes; + (*pSchema)[index].colId = index + 1; + strcpy((*pSchema)[index].name, pExpr->aliasName); index +=1; } } + return TSDB_CODE_SUCCESS; } @@ -1708,8 +1805,8 @@ static void destroyTranslateContext(STranslateContext* pCxt) { } if (NULL != pCxt->pCmdMsg) { - tfree(pCxt->pCmdMsg->pMsg); - tfree(pCxt->pCmdMsg); + taosMemoryFreeClear(pCxt->pCmdMsg->pMsg); + taosMemoryFreeClear(pCxt->pCmdMsg); } taosHashCleanup(pCxt->pDbs); @@ -1870,17 +1967,24 @@ static void toSchema(const SColumnDefNode* pCol, int32_t colId, SSchema* pSchema } static void destroyCreateTbReq(SVCreateTbReq* pReq) { - tfree(pReq->name); - tfree(pReq->ntbCfg.pSchema); + taosMemoryFreeClear(pReq->dbFName); + taosMemoryFreeClear(pReq->name); + taosMemoryFreeClear(pReq->ntbCfg.pSchema); } -static int32_t buildNormalTableBatchReq( - const char* pDbName, const char* pTableName, const SNodeList* pColumns, const SVgroupInfo* pVgroupInfo, SVgroupTablesBatch* pBatch) { +static int32_t buildNormalTableBatchReq(int32_t acctId, const char* pDbName, const char* pTableName, + const SNodeList* pColumns, const SVgroupInfo* pVgroupInfo, SVgroupTablesBatch* pBatch) { + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + SName name = { .type = TSDB_DB_NAME_T, .acctId = acctId }; + strcpy(name.dbname, pDbName); + tNameGetFullDbName(&name, dbFName); + SVCreateTbReq req = {0}; req.type = TD_NORMAL_TABLE; + req.dbFName = strdup(dbFName); req.name = strdup(pTableName); req.ntbCfg.nCols = LIST_LENGTH(pColumns); - req.ntbCfg.pSchema = calloc(req.ntbCfg.nCols, sizeof(SSchema)); + req.ntbCfg.pSchema = taosMemoryCalloc(req.ntbCfg.nCols, sizeof(SSchema)); if (NULL == req.name || NULL == req.ntbCfg.pSchema) { destroyCreateTbReq(&req); return TSDB_CODE_OUT_OF_MEMORY; @@ -1904,9 +2008,9 @@ static int32_t buildNormalTableBatchReq( return TSDB_CODE_SUCCESS; } -static int32_t serializeVgroupTablesBatch(int32_t acctId, SVgroupTablesBatch* pTbBatch, SArray* pBufArray) { +static int32_t serializeVgroupTablesBatch(SVgroupTablesBatch* pTbBatch, SArray* pBufArray) { int tlen = sizeof(SMsgHead) + tSerializeSVCreateTbBatchReq(NULL, &(pTbBatch->req)); - void* buf = malloc(tlen); + void* buf = taosMemoryMalloc(tlen); if (NULL == buf) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1915,7 +2019,7 @@ static int32_t serializeVgroupTablesBatch(int32_t acctId, SVgroupTablesBatch* pT void* pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); tSerializeSVCreateTbBatchReq(&pBuf, &(pTbBatch->req)); - SVgDataBlocks* pVgData = calloc(1, sizeof(SVgDataBlocks)); + SVgDataBlocks* pVgData = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == pVgData) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1932,12 +2036,13 @@ static void destroyCreateTbReqBatch(SVgroupTablesBatch* pTbBatch) { size_t size = taosArrayGetSize(pTbBatch->req.pArray); for(int32_t i = 0; i < size; ++i) { SVCreateTbReq* pTableReq = taosArrayGet(pTbBatch->req.pArray, i); - tfree(pTableReq->name); + taosMemoryFreeClear(pTableReq->dbFName); + taosMemoryFreeClear(pTableReq->name); if (pTableReq->type == TSDB_NORMAL_TABLE) { - tfree(pTableReq->ntbCfg.pSchema); + taosMemoryFreeClear(pTableReq->ntbCfg.pSchema); } else if (pTableReq->type == TSDB_CHILD_TABLE) { - tfree(pTableReq->ctbCfg.pTag); + taosMemoryFreeClear(pTableReq->ctbCfg.pTag); } } @@ -1960,8 +2065,8 @@ static void destroyCreateTbReqArray(SArray* pArray) { size_t size = taosArrayGetSize(pArray); for (size_t i = 0; i < size; ++i) { SVgDataBlocks* pVg = taosArrayGetP(pArray, i); - tfree(pVg->pData); - tfree(pVg); + taosMemoryFreeClear(pVg->pData); + taosMemoryFreeClear(pVg); } taosArrayDestroy(pArray); } @@ -1973,9 +2078,9 @@ static int32_t buildCreateTableDataBlock(int32_t acctId, const SCreateTableStmt* } SVgroupTablesBatch tbatch = {0}; - int32_t code = buildNormalTableBatchReq(pStmt->dbName, pStmt->tableName, pStmt->pCols, pInfo, &tbatch); + int32_t code = buildNormalTableBatchReq(acctId, pStmt->dbName, pStmt->tableName, pStmt->pCols, pInfo, &tbatch); if (TSDB_CODE_SUCCESS == code) { - code = serializeVgroupTablesBatch(acctId, &tbatch, *pBufArray); + code = serializeVgroupTablesBatch(&tbatch, *pBufArray); } destroyCreateTbReqBatch(&tbatch); @@ -2004,9 +2109,16 @@ static int32_t rewriteCreateTable(STranslateContext* pCxt, SQuery* pQuery) { return code; } -static void addCreateTbReqIntoVgroup(SHashObj* pVgroupHashmap, const char* pDbName, const char* pTableName, SKVRow row, uint64_t suid, SVgroupInfo* pVgInfo) { +static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, + const char* pDbName, const char* pTableName, SKVRow row, uint64_t suid, SVgroupInfo* pVgInfo) { + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + SName name = { .type = TSDB_DB_NAME_T, .acctId = acctId }; + strcpy(name.dbname, pDbName); + tNameGetFullDbName(&name, dbFName); + struct SVCreateTbReq req = {0}; req.type = TD_CHILD_TABLE; + req.dbFName = strdup(dbFName); req.name = strdup(pTableName); req.ctbCfg.suid = suid; req.ctbCfg.pTag = row; @@ -2159,10 +2271,10 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla code = getTableHashVgroup(pCxt, pStmt->dbName, pStmt->tableName, &info); } if (TSDB_CODE_SUCCESS == code) { - addCreateTbReqIntoVgroup(pVgroupHashmap, pStmt->dbName, pStmt->tableName, row, pSuperTableMeta->uid, &info); + addCreateTbReqIntoVgroup(pCxt->pParseCxt->acctId, pVgroupHashmap, pStmt->dbName, pStmt->tableName, row, pSuperTableMeta->uid, &info); } - tfree(pSuperTableMeta); + taosMemoryFreeClear(pSuperTableMeta); tdDestroyKVRowBuilder(&kvRowBuilder); return code; } @@ -2181,7 +2293,7 @@ static SArray* serializeVgroupsTablesBatch(int32_t acctId, SHashObj* pVgroupHash break; } - serializeVgroupTablesBatch(acctId, pTbBatch, pBufArray); + serializeVgroupTablesBatch(pTbBatch, pBufArray); destroyCreateTbReqBatch(pTbBatch); } while (true); @@ -2263,7 +2375,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { pQuery->haveResultSet = true; pQuery->directRpc = false; pQuery->msgType = TDMT_VND_QUERY; - code = setReslutSchema(pCxt, pQuery); + code = qExtractResultSchema(pQuery->pRoot, &pQuery->numOfResCols, &pQuery->pResSchema); break; case QUERY_NODE_VNODE_MODIF_STMT: pQuery->haveResultSet = false; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 69c2380b4538765a68881722f5926d4500f4a28b..80d04c5ee4f98ea1c09fedba4c1690bd7fceabef 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -59,6 +59,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Endpoint should be in the format of 'fqdn:port'"; case TSDB_CODE_PAR_EXPRIE_STATEMENT: return "This statement is no longer supported"; + case TSDB_CODE_PAR_INTERVAL_VALUE_TOO_SMALL: + return "This interval value is too small : %s"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: @@ -71,6 +73,7 @@ int32_t generateSyntaxErrMsg(SMsgBuf* pBuf, int32_t errCode, ...) { va_start(vArgList, errCode); vsnprintf(pBuf->buf, pBuf->len, getSyntaxErrFormat(errCode), vArgList); va_end(vArgList); + terrno = errCode; return errCode; } @@ -119,7 +122,7 @@ STableMeta* tableMetaDup(const STableMeta* pTableMeta) { assert(pTableMeta != NULL); size_t size = getTableMetaSize(pTableMeta); - STableMeta* p = malloc(size); + STableMeta* p = taosMemoryMalloc(size); memcpy(p, pTableMeta, size); return p; } diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 08f316b539e02dec5416f2aa0cc3e63f45c0ac3d..92bd6f111b03ccf7b9b3232eefae50a934d61032 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -53,10 +53,10 @@ void qDestroyQuery(SQuery* pQueryNode) { return; } nodesDestroyNode(pQueryNode->pRoot); - tfree(pQueryNode->pResSchema); + taosMemoryFreeClear(pQueryNode->pResSchema); if (NULL != pQueryNode->pCmdMsg) { - tfree(pQueryNode->pCmdMsg->pMsg); - tfree(pQueryNode->pCmdMsg); + taosMemoryFreeClear(pQueryNode->pCmdMsg->pMsg); + taosMemoryFreeClear(pQueryNode->pCmdMsg); } - tfree(pQueryNode); + taosMemoryFreeClear(pQueryNode); } diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 50c29e095fcc722a2edbc58a34e4a6fee76fb7d2..451587c8bf1f42481b3947a5c07b3576a5ed7a84 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -73,7 +73,7 @@ ** which is ParseTOKENTYPE. The entry in the union ** for terminal symbols is called "yy0". ** YYSTACKDEPTH is the maximum depth of the parser's stack. If -** zero the stack is dynamically sized using realloc() +** zero the stack is dynamically sized using taosMemoryRealloc() ** ParseARG_SDECL A static variable declaration for the %extra_argument ** ParseARG_PDECL A parameter declaration for the %extra_argument ** ParseARG_PARAM Code to pass %extra_argument as a subroutine parameter @@ -131,17 +131,17 @@ typedef union { #define ParseCTX_PARAM #define ParseCTX_FETCH #define ParseCTX_STORE -#define YYNSTATE 427 +#define YYNSTATE 430 #define YYNRULE 337 #define YYNTOKEN 163 -#define YY_MAX_SHIFT 426 -#define YY_MIN_SHIFTREDUCE 659 -#define YY_MAX_SHIFTREDUCE 995 -#define YY_ERROR_ACTION 996 -#define YY_ACCEPT_ACTION 997 -#define YY_NO_ACTION 998 -#define YY_MIN_REDUCE 999 -#define YY_MAX_REDUCE 1335 +#define YY_MAX_SHIFT 429 +#define YY_MIN_SHIFTREDUCE 662 +#define YY_MAX_SHIFTREDUCE 998 +#define YY_ERROR_ACTION 999 +#define YY_ACCEPT_ACTION 1000 +#define YY_NO_ACTION 1001 +#define YY_MIN_REDUCE 1002 +#define YY_MAX_REDUCE 1338 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -208,135 +208,135 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (1269) +#define YY_ACTTAB_COUNT (1263) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 1042, 1201, 221, 43, 24, 167, 359, 1197, 1203, 1092, - /* 10 */ 246, 265, 89, 31, 29, 27, 26, 25, 20, 1201, - /* 20 */ 1098, 27, 26, 25, 1081, 1197, 1202, 1103, 31, 29, - /* 30 */ 27, 26, 25, 358, 997, 78, 205, 866, 77, 76, - /* 40 */ 75, 74, 73, 72, 71, 70, 69, 1088, 207, 411, - /* 50 */ 410, 409, 408, 407, 406, 405, 404, 403, 402, 401, - /* 60 */ 400, 399, 398, 397, 396, 395, 394, 393, 1000, 105, - /* 70 */ 266, 1011, 878, 31, 29, 27, 26, 25, 1228, 131, - /* 80 */ 901, 346, 111, 245, 233, 343, 1213, 1177, 271, 78, - /* 90 */ 130, 22, 77, 76, 75, 74, 73, 72, 71, 70, - /* 100 */ 69, 31, 29, 27, 26, 25, 1228, 1314, 207, 287, - /* 110 */ 316, 282, 266, 343, 286, 44, 902, 285, 128, 283, - /* 120 */ 117, 342, 284, 345, 1312, 23, 228, 1189, 896, 897, - /* 130 */ 898, 899, 900, 904, 905, 906, 1079, 200, 1214, 1217, - /* 140 */ 901, 769, 382, 381, 380, 773, 379, 775, 776, 378, - /* 150 */ 778, 375, 109, 784, 372, 786, 787, 369, 366, 1148, - /* 160 */ 1213, 846, 127, 1141, 301, 220, 125, 43, 323, 185, - /* 170 */ 1146, 358, 1133, 30, 28, 938, 902, 844, 254, 238, - /* 180 */ 1228, 230, 392, 846, 1099, 23, 228, 343, 896, 897, - /* 190 */ 898, 899, 900, 904, 905, 906, 1201, 345, 358, 844, - /* 200 */ 893, 1189, 1197, 1202, 302, 359, 331, 845, 12, 118, - /* 210 */ 66, 61, 1214, 1217, 1253, 1213, 962, 275, 206, 1249, - /* 220 */ 1165, 10, 122, 121, 30, 28, 1103, 120, 1314, 845, - /* 230 */ 1314, 1, 230, 423, 846, 1228, 312, 960, 961, 963, - /* 240 */ 964, 117, 330, 117, 1213, 1312, 241, 1312, 10, 321, - /* 250 */ 844, 334, 345, 1168, 1170, 423, 1189, 346, 696, 12, - /* 260 */ 695, 847, 850, 1178, 1228, 359, 62, 1214, 1217, 1253, - /* 270 */ 66, 330, 279, 223, 1249, 112, 278, 281, 697, 327, - /* 280 */ 845, 345, 1, 847, 850, 1189, 1103, 163, 118, 9, - /* 290 */ 8, 59, 1213, 308, 1280, 62, 1214, 1217, 1253, 280, - /* 300 */ 92, 93, 223, 1249, 112, 1314, 423, 327, 1095, 106, - /* 310 */ 1070, 903, 1228, 31, 29, 27, 26, 25, 1313, 343, - /* 320 */ 21, 1213, 1312, 1281, 392, 1267, 90, 868, 92, 345, - /* 330 */ 907, 419, 418, 1189, 847, 850, 114, 1260, 1261, 867, - /* 340 */ 1265, 1228, 1264, 62, 1214, 1217, 1253, 331, 343, 118, - /* 350 */ 223, 1249, 1326, 695, 90, 1094, 1213, 864, 345, 914, - /* 360 */ 1228, 1287, 1189, 865, 160, 1260, 326, 343, 325, 273, - /* 370 */ 234, 1314, 62, 1214, 1217, 1253, 1228, 359, 104, 223, - /* 380 */ 1249, 1326, 1100, 343, 117, 1213, 1105, 385, 1312, 1022, - /* 390 */ 1310, 1090, 320, 345, 30, 28, 1189, 1189, 1103, 1021, - /* 400 */ 1267, 333, 230, 104, 846, 1228, 1080, 62, 1214, 1217, - /* 410 */ 1253, 1106, 343, 1148, 223, 1249, 1326, 1263, 359, 235, - /* 420 */ 844, 1148, 345, 356, 1146, 1271, 1189, 242, 162, 12, - /* 430 */ 1189, 331, 1146, 30, 28, 1012, 195, 1214, 1217, 1103, - /* 440 */ 1189, 230, 1213, 846, 190, 1020, 1019, 1018, 136, 192, - /* 450 */ 845, 134, 1, 1038, 315, 1314, 30, 28, 344, 844, - /* 460 */ 6, 191, 1228, 389, 230, 1213, 846, 388, 117, 343, - /* 470 */ 1017, 123, 1312, 1148, 118, 288, 423, 322, 317, 345, - /* 480 */ 1016, 1015, 844, 1189, 1169, 1228, 1189, 1189, 1189, 845, - /* 490 */ 390, 7, 343, 63, 1214, 1217, 1253, 1148, 869, 1086, - /* 500 */ 1252, 1249, 345, 1045, 847, 850, 1189, 945, 1147, 387, - /* 510 */ 386, 1189, 845, 866, 7, 423, 63, 1214, 1217, 1253, - /* 520 */ 240, 1189, 1189, 341, 1249, 30, 28, 299, 104, 30, - /* 530 */ 28, 64, 384, 230, 338, 846, 1105, 230, 423, 846, - /* 540 */ 297, 1213, 1267, 847, 850, 999, 31, 29, 27, 26, - /* 550 */ 25, 844, 287, 933, 282, 844, 1014, 286, 118, 1262, - /* 560 */ 285, 1228, 283, 118, 1213, 284, 847, 850, 343, 87, - /* 570 */ 86, 85, 84, 83, 82, 81, 80, 79, 345, 864, - /* 580 */ 1033, 845, 1189, 7, 1228, 845, 247, 1, 1071, 259, - /* 590 */ 1013, 343, 107, 1214, 1217, 937, 426, 1189, 260, 149, - /* 600 */ 1213, 345, 290, 164, 359, 1189, 243, 423, 339, 357, - /* 610 */ 183, 423, 304, 88, 104, 63, 1214, 1217, 1253, 415, - /* 620 */ 1228, 182, 1105, 1250, 98, 1103, 1010, 343, 313, 332, - /* 630 */ 1327, 1189, 1031, 1009, 1008, 847, 850, 345, 359, 847, - /* 640 */ 850, 1189, 1142, 180, 229, 359, 60, 1007, 1006, 178, - /* 650 */ 244, 201, 1214, 1217, 293, 1005, 157, 1004, 1213, 1103, - /* 660 */ 1272, 933, 1229, 1213, 272, 258, 1103, 1189, 253, 252, - /* 670 */ 251, 250, 249, 335, 1189, 1189, 1003, 853, 1228, 138, - /* 680 */ 355, 1213, 137, 1228, 140, 343, 1213, 139, 1189, 1189, - /* 690 */ 343, 852, 991, 992, 307, 345, 1189, 146, 1189, 1189, - /* 700 */ 345, 1228, 309, 327, 1189, 328, 1228, 856, 343, 201, - /* 710 */ 1214, 1217, 1002, 343, 107, 1214, 1217, 1189, 345, 9, - /* 720 */ 8, 855, 1189, 345, 92, 227, 864, 1189, 1213, 936, - /* 730 */ 231, 1213, 201, 1214, 1217, 1283, 1213, 201, 1214, 1217, - /* 740 */ 31, 29, 27, 26, 25, 336, 327, 1167, 1228, 52, - /* 750 */ 90, 1228, 1328, 1189, 166, 343, 1228, 1078, 343, 329, - /* 760 */ 113, 1260, 1261, 343, 1265, 345, 1096, 92, 345, 1189, - /* 770 */ 2, 119, 1189, 345, 1213, 142, 256, 1189, 141, 199, - /* 780 */ 1214, 1217, 202, 1214, 1217, 959, 154, 193, 1214, 1217, - /* 790 */ 248, 255, 257, 90, 1228, 261, 1213, 41, 152, 878, - /* 800 */ 908, 343, 1213, 115, 1260, 1261, 872, 1265, 994, 995, - /* 810 */ 262, 345, 32, 263, 389, 1189, 1228, 124, 388, 871, - /* 820 */ 875, 58, 1228, 343, 42, 203, 1214, 1217, 1207, 343, - /* 830 */ 1213, 54, 32, 345, 839, 264, 1213, 1189, 267, 345, - /* 840 */ 1205, 390, 129, 1189, 274, 870, 32, 194, 1214, 1217, - /* 850 */ 1228, 276, 68, 204, 1214, 1217, 1228, 343, 172, 1093, - /* 860 */ 387, 386, 219, 343, 1213, 133, 351, 345, 177, 1089, - /* 870 */ 170, 1189, 135, 345, 762, 100, 101, 1189, 95, 1091, - /* 880 */ 96, 1225, 1214, 1217, 1228, 1087, 98, 1224, 1214, 1217, - /* 890 */ 757, 343, 1213, 102, 790, 794, 103, 800, 1213, 799, - /* 900 */ 306, 345, 41, 303, 145, 1189, 364, 96, 99, 97, - /* 910 */ 305, 98, 1228, 869, 314, 1223, 1214, 1217, 1228, 343, - /* 920 */ 96, 1284, 349, 1294, 150, 343, 311, 850, 1293, 345, - /* 930 */ 153, 222, 1213, 1189, 5, 345, 324, 110, 1274, 1189, - /* 940 */ 1213, 310, 156, 210, 1214, 1217, 237, 236, 4, 209, - /* 950 */ 1214, 1217, 1228, 933, 91, 868, 858, 1268, 33, 343, - /* 960 */ 1228, 159, 158, 340, 1311, 224, 1329, 343, 1213, 345, - /* 970 */ 337, 165, 851, 1189, 292, 214, 17, 345, 1235, 1176, - /* 980 */ 347, 1189, 348, 211, 1214, 1217, 352, 1175, 1228, 300, - /* 990 */ 232, 208, 1214, 1217, 353, 343, 354, 279, 51, 174, - /* 1000 */ 184, 278, 854, 144, 1104, 345, 295, 53, 186, 1189, - /* 1010 */ 362, 289, 181, 215, 143, 213, 212, 422, 277, 198, - /* 1020 */ 1214, 1217, 196, 197, 280, 188, 189, 1183, 360, 822, - /* 1030 */ 1160, 1159, 94, 1158, 1157, 1156, 1155, 1154, 1153, 40, - /* 1040 */ 824, 1152, 39, 1151, 1150, 1149, 1044, 1182, 1173, 126, - /* 1050 */ 1082, 708, 1043, 1041, 268, 269, 859, 850, 270, 1030, - /* 1060 */ 1029, 1026, 1084, 67, 132, 1083, 805, 804, 1039, 1034, - /* 1070 */ 737, 803, 1032, 1025, 1024, 1181, 65, 736, 735, 216, - /* 1080 */ 734, 1180, 36, 1172, 45, 217, 218, 148, 296, 733, - /* 1090 */ 3, 32, 732, 298, 291, 14, 294, 151, 15, 34, - /* 1100 */ 37, 11, 48, 319, 1205, 8, 19, 161, 894, 350, - /* 1110 */ 147, 176, 998, 998, 998, 998, 998, 318, 1171, 998, - /* 1120 */ 783, 998, 998, 998, 998, 998, 998, 998, 998, 980, - /* 1130 */ 979, 225, 984, 983, 226, 998, 998, 998, 998, 998, - /* 1140 */ 998, 958, 998, 175, 998, 998, 108, 155, 998, 952, - /* 1150 */ 46, 998, 860, 951, 47, 768, 930, 929, 998, 985, - /* 1160 */ 998, 998, 998, 998, 998, 998, 998, 998, 998, 13, - /* 1170 */ 35, 116, 876, 16, 18, 998, 169, 956, 171, 168, - /* 1180 */ 49, 173, 50, 998, 1204, 363, 239, 54, 367, 38, - /* 1190 */ 1040, 179, 370, 373, 998, 998, 998, 998, 376, 998, - /* 1200 */ 998, 796, 998, 791, 365, 998, 788, 728, 720, 998, - /* 1210 */ 420, 848, 361, 785, 368, 798, 797, 782, 998, 998, - /* 1220 */ 727, 371, 998, 998, 779, 726, 55, 374, 706, 777, - /* 1230 */ 391, 725, 724, 1028, 414, 1027, 1023, 723, 413, 998, - /* 1240 */ 377, 998, 722, 721, 56, 57, 719, 718, 187, 729, - /* 1250 */ 421, 717, 716, 715, 714, 713, 712, 383, 424, 711, - /* 1260 */ 412, 425, 998, 416, 417, 998, 998, 781, 780, + /* 0 */ 1045, 1204, 225, 43, 24, 170, 362, 1200, 1206, 1095, + /* 10 */ 250, 269, 89, 31, 29, 27, 26, 25, 20, 1204, + /* 20 */ 1101, 27, 26, 25, 1084, 1200, 1205, 1106, 31, 29, + /* 30 */ 27, 26, 25, 361, 1000, 78, 209, 869, 77, 76, + /* 40 */ 75, 74, 73, 72, 71, 70, 69, 1091, 211, 414, + /* 50 */ 413, 412, 411, 410, 409, 408, 407, 406, 405, 404, + /* 60 */ 403, 402, 401, 400, 399, 398, 397, 396, 1003, 105, + /* 70 */ 270, 1014, 881, 31, 29, 27, 26, 25, 1231, 132, + /* 80 */ 904, 349, 111, 249, 237, 346, 1216, 1180, 275, 78, + /* 90 */ 131, 22, 77, 76, 75, 74, 73, 72, 71, 70, + /* 100 */ 69, 31, 29, 27, 26, 25, 1231, 1317, 211, 291, + /* 110 */ 320, 286, 270, 346, 290, 44, 905, 289, 129, 287, + /* 120 */ 117, 345, 288, 348, 1315, 23, 232, 1192, 899, 900, + /* 130 */ 901, 902, 903, 907, 908, 909, 1082, 204, 1217, 1220, + /* 140 */ 904, 772, 385, 384, 383, 776, 382, 778, 779, 381, + /* 150 */ 781, 378, 109, 787, 375, 789, 790, 372, 369, 1151, + /* 160 */ 1216, 849, 128, 1144, 305, 224, 125, 43, 326, 189, + /* 170 */ 1149, 361, 1136, 30, 28, 941, 905, 847, 258, 242, + /* 180 */ 1231, 234, 395, 849, 1102, 23, 232, 346, 899, 900, + /* 190 */ 901, 902, 903, 907, 908, 909, 1204, 348, 361, 847, + /* 200 */ 896, 1192, 1200, 1205, 306, 362, 334, 848, 12, 118, + /* 210 */ 66, 61, 1217, 1220, 1256, 1216, 965, 279, 210, 1252, + /* 220 */ 1168, 10, 122, 121, 30, 28, 1106, 120, 1317, 848, + /* 230 */ 1317, 1, 234, 426, 849, 1231, 316, 963, 964, 966, + /* 240 */ 967, 117, 333, 117, 1216, 1315, 245, 1315, 10, 324, + /* 250 */ 847, 337, 348, 1171, 1173, 426, 1192, 349, 699, 12, + /* 260 */ 698, 850, 853, 1181, 1231, 362, 62, 1217, 1220, 1256, + /* 270 */ 66, 333, 283, 227, 1252, 112, 282, 285, 700, 330, + /* 280 */ 848, 348, 1, 850, 853, 1192, 1106, 166, 118, 9, + /* 290 */ 8, 59, 1216, 312, 1283, 62, 1217, 1220, 1256, 284, + /* 300 */ 92, 93, 227, 1252, 112, 1317, 426, 330, 1098, 106, + /* 310 */ 1073, 906, 1231, 31, 29, 27, 26, 25, 1316, 346, + /* 320 */ 21, 1216, 1315, 1284, 395, 1270, 90, 871, 92, 348, + /* 330 */ 910, 422, 421, 1192, 850, 853, 114, 1263, 1264, 870, + /* 340 */ 1268, 1231, 1267, 62, 1217, 1220, 1256, 334, 346, 118, + /* 350 */ 227, 1252, 1329, 698, 90, 1097, 1216, 867, 348, 917, + /* 360 */ 1231, 1290, 1192, 868, 163, 1263, 329, 346, 328, 277, + /* 370 */ 238, 1317, 62, 1217, 1220, 1256, 1231, 362, 104, 227, + /* 380 */ 1252, 1329, 1103, 346, 117, 1216, 1108, 388, 1315, 1025, + /* 390 */ 1313, 1093, 323, 348, 30, 28, 1192, 1192, 1106, 1024, + /* 400 */ 1270, 336, 234, 104, 849, 1231, 1083, 62, 1217, 1220, + /* 410 */ 1256, 1109, 346, 1151, 227, 1252, 1329, 1266, 362, 239, + /* 420 */ 847, 1151, 348, 359, 1149, 1274, 1192, 246, 165, 12, + /* 430 */ 1192, 334, 1149, 30, 28, 1015, 199, 1217, 1220, 1106, + /* 440 */ 1192, 234, 1216, 849, 194, 1023, 1022, 1021, 137, 196, + /* 450 */ 848, 135, 1, 1041, 319, 1317, 30, 28, 347, 847, + /* 460 */ 6, 195, 1231, 392, 234, 1216, 849, 391, 117, 346, + /* 470 */ 1020, 123, 1315, 1151, 118, 292, 426, 325, 321, 348, + /* 480 */ 1019, 1018, 847, 1192, 1172, 1231, 1192, 1192, 1192, 848, + /* 490 */ 393, 7, 346, 63, 1217, 1220, 1256, 1151, 872, 1089, + /* 500 */ 1255, 1252, 348, 1048, 850, 853, 1192, 948, 1150, 390, + /* 510 */ 389, 1192, 848, 869, 7, 426, 63, 1217, 1220, 1256, + /* 520 */ 244, 1192, 1192, 344, 1252, 30, 28, 303, 104, 30, + /* 530 */ 28, 64, 387, 234, 341, 849, 1108, 234, 426, 849, + /* 540 */ 301, 1216, 1270, 850, 853, 1002, 31, 29, 27, 26, + /* 550 */ 25, 847, 291, 936, 286, 847, 1017, 290, 118, 1265, + /* 560 */ 289, 1231, 287, 118, 1216, 288, 850, 853, 346, 87, + /* 570 */ 86, 85, 84, 83, 82, 81, 80, 79, 348, 867, + /* 580 */ 1036, 848, 1192, 7, 1231, 848, 251, 1, 1074, 263, + /* 590 */ 1016, 346, 107, 1217, 1220, 940, 429, 1192, 264, 151, + /* 600 */ 1216, 348, 294, 167, 362, 1192, 247, 426, 342, 360, + /* 610 */ 187, 426, 308, 88, 104, 63, 1217, 1220, 1256, 418, + /* 620 */ 1231, 186, 1108, 1253, 98, 1106, 1013, 346, 317, 335, + /* 630 */ 1330, 1192, 1034, 1012, 1011, 850, 853, 348, 362, 850, + /* 640 */ 853, 1192, 1145, 184, 233, 362, 60, 1010, 1009, 182, + /* 650 */ 248, 205, 1217, 1220, 297, 1008, 160, 1007, 1216, 1106, + /* 660 */ 1275, 936, 1232, 1216, 276, 262, 1106, 1192, 257, 256, + /* 670 */ 255, 254, 253, 338, 1192, 1192, 1006, 856, 1231, 139, + /* 680 */ 358, 1216, 138, 1231, 141, 346, 1216, 140, 1192, 1192, + /* 690 */ 346, 855, 994, 995, 311, 348, 1192, 147, 1192, 1192, + /* 700 */ 348, 1231, 313, 330, 1192, 331, 1231, 859, 346, 205, + /* 710 */ 1217, 1220, 1005, 346, 107, 1217, 1220, 1192, 348, 9, + /* 720 */ 8, 858, 1192, 348, 92, 231, 867, 1192, 1216, 939, + /* 730 */ 235, 1216, 205, 1217, 1220, 1286, 1216, 205, 1217, 1220, + /* 740 */ 31, 29, 27, 26, 25, 339, 330, 1170, 1231, 52, + /* 750 */ 90, 1231, 1331, 1192, 169, 346, 1231, 1081, 346, 332, + /* 760 */ 113, 1263, 1264, 346, 1268, 348, 1099, 92, 348, 1192, + /* 770 */ 2, 119, 1192, 348, 1216, 143, 260, 1192, 142, 203, + /* 780 */ 1217, 1220, 206, 1217, 1220, 962, 156, 197, 1217, 1220, + /* 790 */ 252, 259, 261, 90, 1231, 265, 1216, 41, 154, 881, + /* 800 */ 911, 346, 1216, 115, 1263, 1264, 875, 1268, 997, 998, + /* 810 */ 266, 348, 32, 267, 392, 1192, 1231, 124, 391, 874, + /* 820 */ 878, 58, 1231, 346, 127, 207, 1217, 1220, 1210, 346, + /* 830 */ 1216, 54, 32, 348, 842, 268, 1216, 1192, 271, 348, + /* 840 */ 1208, 393, 42, 1192, 278, 130, 32, 198, 1217, 1220, + /* 850 */ 1231, 873, 68, 208, 1217, 1220, 1231, 346, 175, 280, + /* 860 */ 390, 389, 223, 346, 1216, 1096, 354, 348, 181, 134, + /* 870 */ 173, 1192, 1092, 348, 765, 136, 100, 1192, 95, 101, + /* 880 */ 96, 1228, 1217, 1220, 1231, 1094, 98, 1227, 1217, 1220, + /* 890 */ 760, 346, 1216, 1090, 793, 797, 102, 803, 1216, 802, + /* 900 */ 310, 348, 41, 103, 146, 1192, 367, 96, 99, 97, + /* 910 */ 149, 98, 1231, 307, 309, 1226, 1217, 1220, 1231, 346, + /* 920 */ 96, 872, 318, 1297, 1287, 346, 352, 152, 853, 348, + /* 930 */ 315, 1296, 1216, 1192, 226, 348, 5, 159, 155, 1192, + /* 940 */ 1216, 322, 327, 214, 1217, 1220, 241, 240, 1277, 213, + /* 950 */ 1217, 1220, 1231, 314, 4, 161, 861, 936, 91, 346, + /* 960 */ 1231, 871, 110, 1271, 33, 162, 228, 346, 1216, 348, + /* 970 */ 1332, 340, 854, 1192, 296, 218, 343, 348, 17, 1238, + /* 980 */ 1179, 1192, 168, 215, 1217, 1220, 350, 1314, 1231, 304, + /* 990 */ 351, 212, 1217, 1220, 355, 346, 1178, 283, 356, 236, + /* 1000 */ 177, 282, 857, 145, 357, 348, 299, 179, 53, 1192, + /* 1010 */ 51, 293, 188, 219, 144, 217, 216, 190, 281, 202, + /* 1020 */ 1217, 1220, 1107, 365, 284, 185, 425, 200, 363, 201, + /* 1030 */ 192, 193, 1186, 825, 1163, 1162, 94, 1161, 1160, 40, + /* 1040 */ 1159, 1158, 39, 1157, 1156, 827, 1155, 1154, 1153, 1152, + /* 1050 */ 1047, 1185, 1176, 126, 1085, 711, 862, 853, 1046, 1044, + /* 1060 */ 272, 273, 274, 1033, 1032, 1029, 1087, 67, 133, 808, + /* 1070 */ 1086, 807, 1042, 1037, 740, 806, 1035, 739, 1028, 220, + /* 1080 */ 738, 1027, 221, 737, 222, 1184, 1183, 36, 1175, 150, + /* 1090 */ 300, 302, 3, 736, 735, 65, 45, 32, 14, 153, + /* 1100 */ 295, 37, 15, 158, 19, 298, 1208, 34, 11, 164, + /* 1110 */ 48, 8, 35, 16, 148, 897, 353, 1174, 180, 1001, + /* 1120 */ 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, + /* 1130 */ 983, 982, 229, 987, 986, 230, 863, 1001, 1001, 1001, + /* 1140 */ 1001, 1001, 1001, 178, 1001, 961, 1001, 108, 1001, 157, + /* 1150 */ 1001, 955, 46, 1001, 954, 933, 1001, 47, 1001, 1001, + /* 1160 */ 932, 1001, 988, 1001, 1001, 1001, 1001, 1001, 1001, 366, + /* 1170 */ 879, 13, 116, 18, 243, 172, 959, 174, 176, 1001, + /* 1180 */ 171, 49, 50, 1001, 1207, 38, 370, 1001, 373, 794, + /* 1190 */ 54, 376, 771, 1001, 379, 183, 1001, 364, 799, 1001, + /* 1200 */ 1001, 1001, 368, 731, 791, 371, 1043, 1001, 788, 723, + /* 1210 */ 1001, 428, 423, 801, 374, 786, 800, 782, 1001, 1001, + /* 1220 */ 1001, 730, 377, 729, 728, 709, 780, 727, 380, 394, + /* 1230 */ 785, 726, 1031, 725, 724, 417, 722, 721, 1030, 1026, + /* 1240 */ 416, 720, 55, 56, 57, 415, 420, 732, 719, 424, + /* 1250 */ 1001, 851, 718, 717, 716, 386, 715, 784, 191, 783, + /* 1260 */ 714, 419, 427, }; static const YYCODETYPE yy_lookahead[] = { /* 0 */ 0, 207, 190, 174, 221, 222, 172, 213, 214, 187, @@ -423,49 +423,49 @@ static const YYCODETYPE yy_lookahead[] = { /* 810 */ 211, 203, 83, 193, 57, 207, 186, 174, 61, 20, /* 820 */ 71, 70, 186, 193, 174, 217, 218, 219, 70, 193, /* 830 */ 166, 80, 83, 203, 71, 204, 166, 207, 172, 203, - /* 840 */ 82, 84, 174, 207, 168, 20, 83, 217, 218, 219, - /* 850 */ 186, 186, 172, 217, 218, 219, 186, 193, 71, 186, + /* 840 */ 82, 84, 174, 207, 168, 174, 83, 217, 218, 219, + /* 850 */ 186, 20, 172, 217, 218, 219, 186, 193, 71, 186, /* 860 */ 103, 104, 168, 193, 166, 186, 71, 203, 71, 186, /* 870 */ 83, 207, 186, 203, 71, 186, 186, 207, 83, 186, /* 880 */ 83, 217, 218, 219, 186, 186, 83, 217, 218, 219, /* 890 */ 71, 193, 166, 186, 71, 71, 186, 71, 166, 71, - /* 900 */ 204, 203, 83, 211, 171, 207, 83, 83, 71, 83, - /* 910 */ 193, 83, 186, 20, 145, 217, 218, 219, 186, 193, - /* 920 */ 83, 216, 144, 246, 208, 193, 207, 123, 246, 203, - /* 930 */ 208, 207, 166, 207, 152, 203, 151, 240, 243, 207, - /* 940 */ 166, 140, 242, 217, 218, 219, 12, 13, 139, 217, - /* 950 */ 218, 219, 186, 136, 193, 20, 22, 215, 114, 193, - /* 960 */ 186, 227, 239, 156, 252, 160, 257, 193, 166, 203, - /* 970 */ 154, 251, 38, 207, 4, 35, 70, 203, 223, 208, - /* 980 */ 207, 207, 207, 217, 218, 219, 118, 208, 186, 19, - /* 990 */ 207, 217, 218, 219, 205, 193, 204, 57, 171, 193, - /* 1000 */ 182, 61, 68, 33, 193, 203, 36, 70, 172, 207, - /* 1010 */ 189, 41, 171, 73, 44, 75, 76, 168, 78, 217, - /* 1020 */ 218, 219, 180, 180, 84, 173, 164, 0, 94, 82, - /* 1030 */ 0, 0, 114, 0, 0, 0, 0, 0, 0, 69, - /* 1040 */ 22, 0, 72, 0, 0, 0, 0, 0, 0, 43, - /* 1050 */ 0, 48, 0, 0, 38, 36, 122, 123, 43, 0, - /* 1060 */ 0, 0, 0, 79, 77, 0, 38, 38, 0, 0, - /* 1070 */ 38, 22, 0, 0, 0, 0, 20, 38, 38, 22, - /* 1080 */ 38, 0, 121, 0, 70, 22, 22, 116, 22, 38, - /* 1090 */ 83, 83, 38, 22, 39, 141, 38, 71, 141, 135, - /* 1100 */ 83, 141, 4, 83, 82, 2, 83, 82, 126, 119, - /* 1110 */ 43, 116, 258, 258, 258, 258, 258, 38, 0, 258, - /* 1120 */ 96, 258, 258, 258, 258, 258, 258, 258, 258, 38, - /* 1130 */ 38, 38, 38, 38, 38, 258, 258, 258, 258, 258, - /* 1140 */ 258, 71, 258, 43, 258, 258, 70, 70, 258, 71, - /* 1150 */ 70, 258, 22, 71, 70, 22, 71, 71, 258, 71, - /* 1160 */ 258, 258, 258, 258, 258, 258, 258, 258, 258, 70, - /* 1170 */ 83, 82, 71, 83, 70, 258, 71, 71, 70, 82, - /* 1180 */ 70, 70, 70, 258, 82, 38, 38, 80, 38, 70, - /* 1190 */ 0, 82, 38, 38, 258, 258, 258, 258, 38, 258, - /* 1200 */ 258, 22, 258, 71, 70, 258, 71, 22, 22, 258, - /* 1210 */ 22, 22, 81, 71, 70, 38, 38, 96, 258, 258, - /* 1220 */ 38, 70, 258, 258, 71, 38, 70, 70, 48, 71, - /* 1230 */ 47, 38, 38, 0, 43, 0, 0, 38, 36, 258, - /* 1240 */ 70, 258, 38, 38, 70, 70, 38, 38, 22, 68, - /* 1250 */ 21, 38, 38, 38, 38, 38, 38, 84, 21, 38, - /* 1260 */ 38, 20, 258, 38, 37, 258, 258, 96, 96, 258, + /* 900 */ 204, 203, 83, 186, 171, 207, 83, 83, 71, 83, + /* 910 */ 171, 83, 186, 211, 193, 217, 218, 219, 186, 193, + /* 920 */ 83, 20, 145, 246, 216, 193, 144, 208, 123, 203, + /* 930 */ 207, 246, 166, 207, 207, 203, 152, 242, 208, 207, + /* 940 */ 166, 207, 151, 217, 218, 219, 12, 13, 243, 217, + /* 950 */ 218, 219, 186, 140, 139, 239, 22, 136, 193, 193, + /* 960 */ 186, 20, 240, 215, 114, 227, 160, 193, 166, 203, + /* 970 */ 257, 154, 38, 207, 4, 35, 156, 203, 70, 223, + /* 980 */ 208, 207, 251, 217, 218, 219, 207, 252, 186, 19, + /* 990 */ 207, 217, 218, 219, 118, 193, 208, 57, 205, 207, + /* 1000 */ 193, 61, 68, 33, 204, 203, 36, 171, 70, 207, + /* 1010 */ 171, 41, 182, 73, 44, 75, 76, 172, 78, 217, + /* 1020 */ 218, 219, 193, 189, 84, 171, 168, 180, 94, 180, + /* 1030 */ 173, 164, 0, 82, 0, 0, 114, 0, 0, 69, + /* 1040 */ 0, 0, 72, 0, 0, 22, 0, 0, 0, 0, + /* 1050 */ 0, 0, 0, 43, 0, 48, 122, 123, 0, 0, + /* 1060 */ 38, 36, 43, 0, 0, 0, 0, 79, 77, 38, + /* 1070 */ 0, 38, 0, 0, 38, 22, 0, 38, 0, 22, + /* 1080 */ 38, 0, 22, 38, 22, 0, 0, 121, 0, 116, + /* 1090 */ 22, 22, 83, 38, 38, 20, 70, 83, 141, 71, + /* 1100 */ 39, 83, 141, 83, 83, 38, 82, 135, 141, 82, + /* 1110 */ 4, 2, 83, 83, 43, 126, 119, 0, 116, 258, + /* 1120 */ 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, + /* 1130 */ 38, 38, 38, 38, 38, 38, 22, 258, 258, 258, + /* 1140 */ 258, 258, 258, 43, 258, 71, 258, 70, 258, 70, + /* 1150 */ 258, 71, 70, 258, 71, 71, 258, 70, 258, 258, + /* 1160 */ 71, 258, 71, 258, 258, 258, 258, 258, 258, 38, + /* 1170 */ 71, 70, 82, 70, 38, 71, 71, 70, 70, 258, + /* 1180 */ 82, 70, 70, 258, 82, 70, 38, 258, 38, 71, + /* 1190 */ 80, 38, 22, 258, 38, 82, 258, 81, 22, 258, + /* 1200 */ 258, 258, 70, 22, 71, 70, 0, 258, 71, 22, + /* 1210 */ 258, 20, 22, 38, 70, 96, 38, 71, 258, 258, + /* 1220 */ 258, 38, 70, 38, 38, 48, 71, 38, 70, 47, + /* 1230 */ 96, 38, 0, 38, 38, 43, 38, 38, 0, 0, + /* 1240 */ 36, 38, 70, 70, 70, 38, 37, 68, 38, 21, + /* 1250 */ 258, 22, 38, 38, 38, 84, 38, 96, 22, 96, + /* 1260 */ 38, 38, 21, 258, 258, 258, 258, 258, 258, 258, /* 1270 */ 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, /* 1280 */ 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, /* 1290 */ 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, @@ -481,10 +481,9 @@ static const YYCODETYPE yy_lookahead[] = { /* 1390 */ 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, /* 1400 */ 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, /* 1410 */ 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, - /* 1420 */ 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, - /* 1430 */ 258, 258, + /* 1420 */ 258, 258, 258, 258, 258, 258, }; -#define YY_SHIFT_COUNT (426) +#define YY_SHIFT_COUNT (429) #define YY_SHIFT_MIN (0) #define YY_SHIFT_MAX (1241) static const unsigned short int yy_shift_ofst[] = { @@ -500,41 +499,41 @@ static const unsigned short int yy_shift_ofst[] = { /* 90 */ 307, 307, 307, 136, 343, 319, 17, 17, 17, 306, /* 100 */ 56, 56, 56, 56, 56, 68, 503, 534, 90, 215, /* 110 */ 335, 331, 478, 525, 417, 525, 493, 248, 591, 706, - /* 120 */ 657, 661, 661, 706, 786, 66, 343, 799, 66, 706, - /* 130 */ 66, 825, 17, 17, 17, 17, 17, 17, 17, 17, - /* 140 */ 17, 17, 17, 706, 825, 786, 278, 343, 799, 893, - /* 150 */ 769, 778, 804, 769, 778, 804, 782, 785, 801, 809, - /* 160 */ 817, 343, 935, 844, 805, 807, 816, 906, 17, 778, - /* 170 */ 804, 804, 778, 804, 868, 343, 799, 306, 278, 343, - /* 180 */ 937, 706, 278, 825, 1269, 1269, 1269, 1269, 0, 545, - /* 190 */ 577, 46, 970, 16, 89, 728, 406, 757, 301, 301, - /* 200 */ 301, 301, 301, 301, 301, 115, 288, 196, 7, 7, - /* 210 */ 7, 7, 374, 605, 610, 701, 453, 580, 632, 506, - /* 220 */ 541, 714, 715, 718, 647, 590, 451, 729, 74, 749, - /* 230 */ 758, 763, 787, 795, 797, 803, 639, 653, 819, 823, - /* 240 */ 824, 826, 828, 837, 751, 1027, 947, 1030, 1031, 918, - /* 250 */ 1033, 1034, 1035, 1036, 1037, 1038, 1018, 1041, 1043, 1044, - /* 260 */ 1045, 1046, 1047, 1048, 1006, 1050, 1003, 1052, 1053, 1016, - /* 270 */ 1019, 1015, 1059, 1060, 1061, 1062, 984, 987, 1028, 1029, - /* 280 */ 1049, 1065, 1032, 1039, 1040, 1042, 1051, 1054, 1068, 1057, - /* 290 */ 1069, 1063, 1055, 1072, 1064, 1058, 1073, 1066, 1074, 1071, - /* 300 */ 1056, 1075, 1081, 961, 1083, 1014, 1067, 971, 1007, 1008, - /* 310 */ 954, 1026, 1017, 1070, 1076, 1077, 1078, 1080, 1082, 1079, - /* 320 */ 1020, 1022, 1084, 1023, 957, 1085, 1086, 1025, 964, 1087, - /* 330 */ 1089, 1088, 1090, 960, 1098, 1091, 1092, 1093, 1094, 1095, - /* 340 */ 1096, 1103, 982, 1097, 1101, 1099, 1104, 1105, 1106, 1108, - /* 350 */ 1111, 990, 1110, 1118, 1100, 995, 1112, 1107, 1102, 1109, - /* 360 */ 1130, 1119, 1131, 1132, 1147, 1148, 1134, 1135, 1150, 1144, - /* 370 */ 1142, 1154, 1151, 1153, 1155, 1157, 1158, 1160, 1170, 1024, - /* 380 */ 1121, 1171, 1172, 1133, 1173, 1156, 1174, 1175, 1177, 1178, - /* 390 */ 1179, 1180, 1183, 1181, 1185, 1182, 1187, 1193, 1194, 1199, - /* 400 */ 1204, 1205, 1186, 1208, 1209, 1213, 1214, 1215, 1216, 1217, - /* 410 */ 1218, 1221, 1190, 1222, 1202, 1191, 1233, 1225, 1227, 1235, - /* 420 */ 1236, 1188, 1229, 1189, 1226, 1237, 1241, + /* 120 */ 657, 661, 661, 706, 786, 66, 343, 799, 66, 66, + /* 130 */ 706, 66, 831, 17, 17, 17, 17, 17, 17, 17, + /* 140 */ 17, 17, 17, 17, 706, 831, 786, 278, 343, 799, + /* 150 */ 278, 901, 777, 782, 805, 777, 782, 805, 805, 784, + /* 160 */ 791, 813, 815, 821, 343, 941, 850, 806, 820, 817, + /* 170 */ 908, 17, 782, 805, 805, 782, 805, 876, 343, 799, + /* 180 */ 278, 306, 278, 343, 938, 706, 278, 831, 1263, 1263, + /* 190 */ 1263, 1263, 0, 545, 577, 46, 970, 16, 89, 728, + /* 200 */ 406, 757, 301, 301, 301, 301, 301, 301, 301, 115, + /* 210 */ 288, 196, 7, 7, 7, 7, 374, 605, 610, 701, + /* 220 */ 453, 580, 632, 506, 541, 714, 715, 718, 647, 590, + /* 230 */ 451, 729, 74, 749, 758, 763, 787, 795, 797, 803, + /* 240 */ 639, 653, 819, 823, 824, 826, 828, 837, 751, 1032, + /* 250 */ 951, 1034, 1035, 922, 1037, 1038, 1040, 1041, 1043, 1044, + /* 260 */ 1023, 1046, 1047, 1048, 1049, 1050, 1051, 1052, 1010, 1054, + /* 270 */ 1007, 1058, 1059, 1022, 1025, 1019, 1063, 1064, 1065, 1066, + /* 280 */ 988, 991, 1031, 1033, 1053, 1070, 1036, 1039, 1042, 1045, + /* 290 */ 1055, 1056, 1072, 1057, 1073, 1060, 1061, 1076, 1062, 1067, + /* 300 */ 1078, 1068, 1081, 1069, 1075, 1085, 1086, 966, 1088, 1026, + /* 310 */ 1071, 973, 1009, 1014, 957, 1028, 1018, 1074, 1077, 1079, + /* 320 */ 1080, 1082, 1083, 1020, 1024, 1087, 1021, 961, 1084, 1089, + /* 330 */ 1027, 972, 1029, 1090, 1091, 1030, 967, 1106, 1092, 1093, + /* 340 */ 1094, 1095, 1096, 1097, 1109, 989, 1098, 1099, 1101, 1103, + /* 350 */ 1104, 1105, 1107, 1108, 997, 1111, 1117, 1100, 1002, 1112, + /* 360 */ 1110, 1102, 1113, 1114, 1115, 1116, 1118, 1131, 1136, 1132, + /* 370 */ 1133, 1148, 1135, 1137, 1150, 1144, 1146, 1153, 1152, 1155, + /* 380 */ 1156, 1158, 1119, 1134, 1161, 1163, 1170, 1171, 1172, 1173, + /* 390 */ 1174, 1175, 1178, 1176, 1177, 1182, 1179, 1181, 1183, 1185, + /* 400 */ 1186, 1189, 1193, 1195, 1196, 1187, 1198, 1199, 1203, 1210, + /* 410 */ 1214, 1215, 1216, 1218, 1222, 1206, 1207, 1204, 1192, 1232, + /* 420 */ 1223, 1209, 1238, 1239, 1190, 1228, 1229, 1236, 1241, 1191, }; -#define YY_REDUCE_COUNT (187) +#define YY_REDUCE_COUNT (191) #define YY_REDUCE_MIN (-217) -#define YY_REDUCE_MAX (862) +#define YY_REDUCE_MAX (867) static const short yy_reduce_ofst[] = { /* 0 */ -129, -6, 49, 78, 126, 155, 190, 219, 276, 299, /* 10 */ 135, 375, 398, 434, 492, -80, 497, 515, 520, 562, @@ -548,58 +547,59 @@ static const short yy_reduce_ofst[] = { /* 90 */ 110, 185, 327, 578, 27, 54, 217, 287, 311, -10, /* 100 */ -178, -140, 204, 312, 345, 268, 412, 347, 381, 446, /* 110 */ 415, 495, 519, 471, 471, 471, 476, 504, 533, 575, - /* 120 */ 589, 592, 593, 623, 599, 643, 620, 631, 650, 666, - /* 130 */ 668, 676, 665, 673, 679, 683, 686, 689, 690, 693, - /* 140 */ 699, 707, 710, 680, 694, 692, 733, 717, 696, 705, - /* 150 */ 677, 716, 719, 682, 722, 724, 695, 700, 697, 723, - /* 160 */ 471, 761, 742, 734, 709, 712, 720, 755, 476, 771, - /* 170 */ 773, 775, 779, 783, 789, 806, 792, 818, 827, 811, - /* 180 */ 821, 836, 841, 849, 842, 843, 852, 862, + /* 120 */ 589, 592, 593, 623, 599, 643, 620, 631, 650, 668, + /* 130 */ 666, 671, 676, 673, 679, 683, 686, 689, 690, 693, + /* 140 */ 699, 707, 710, 717, 680, 694, 702, 733, 721, 696, + /* 150 */ 739, 708, 677, 719, 723, 685, 730, 727, 734, 705, + /* 160 */ 695, 722, 716, 471, 765, 748, 738, 713, 735, 731, + /* 170 */ 756, 476, 772, 779, 783, 788, 792, 793, 807, 800, + /* 180 */ 836, 830, 839, 829, 834, 845, 854, 858, 847, 849, + /* 190 */ 857, 867, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 996, 996, 996, 996, 996, 996, 996, 996, 996, 996, - /* 10 */ 996, 996, 996, 996, 996, 996, 996, 996, 996, 996, - /* 20 */ 996, 996, 996, 996, 996, 996, 996, 996, 996, 996, - /* 30 */ 996, 996, 996, 996, 996, 996, 996, 996, 996, 996, - /* 40 */ 996, 996, 996, 996, 1049, 996, 996, 996, 996, 996, - /* 50 */ 996, 996, 996, 996, 996, 996, 996, 996, 996, 996, - /* 60 */ 1047, 996, 1255, 996, 1161, 996, 996, 996, 996, 996, - /* 70 */ 996, 996, 996, 996, 996, 996, 996, 996, 996, 996, - /* 80 */ 996, 996, 996, 996, 996, 996, 996, 996, 996, 1049, - /* 90 */ 1266, 1266, 1266, 1047, 996, 996, 996, 996, 996, 1132, - /* 100 */ 996, 996, 996, 996, 996, 996, 996, 1330, 996, 1085, - /* 110 */ 1290, 996, 1282, 1258, 1272, 1259, 996, 1315, 1275, 996, - /* 120 */ 1166, 1163, 1163, 996, 996, 1049, 996, 996, 1049, 996, - /* 130 */ 1049, 996, 996, 996, 996, 996, 996, 996, 996, 996, - /* 140 */ 996, 996, 996, 996, 996, 996, 1047, 996, 996, 996, - /* 150 */ 1297, 1295, 996, 1297, 1295, 996, 1309, 1305, 1288, 1286, - /* 160 */ 1272, 996, 996, 996, 1333, 1321, 1317, 996, 996, 1295, - /* 170 */ 996, 996, 1295, 996, 1174, 996, 996, 996, 1047, 996, - /* 180 */ 1101, 996, 1047, 996, 1135, 1135, 1050, 1001, 996, 996, - /* 190 */ 996, 996, 996, 996, 996, 996, 996, 996, 1227, 1308, - /* 200 */ 1307, 1226, 1232, 1231, 1230, 996, 996, 996, 1221, 1222, - /* 210 */ 1220, 1219, 996, 996, 996, 996, 996, 996, 996, 996, - /* 220 */ 996, 996, 996, 1256, 996, 1318, 1322, 996, 996, 996, - /* 230 */ 1206, 996, 996, 996, 996, 996, 996, 996, 996, 996, - /* 240 */ 996, 996, 996, 996, 996, 996, 996, 996, 996, 996, - /* 250 */ 996, 996, 996, 996, 996, 996, 996, 996, 996, 996, - /* 260 */ 996, 996, 996, 996, 996, 996, 996, 996, 996, 996, - /* 270 */ 996, 996, 996, 996, 996, 996, 996, 996, 996, 996, - /* 280 */ 996, 996, 996, 996, 996, 996, 996, 996, 996, 996, - /* 290 */ 996, 996, 996, 996, 996, 996, 996, 996, 996, 996, - /* 300 */ 996, 996, 996, 996, 996, 996, 996, 996, 1279, 1289, - /* 310 */ 996, 996, 996, 996, 996, 996, 996, 996, 996, 996, - /* 320 */ 996, 1206, 996, 1306, 996, 1265, 1261, 996, 996, 1257, - /* 330 */ 996, 996, 1316, 996, 996, 996, 996, 996, 996, 996, - /* 340 */ 996, 1251, 996, 996, 996, 996, 996, 996, 996, 996, - /* 350 */ 996, 996, 996, 996, 996, 996, 996, 996, 1205, 996, - /* 360 */ 996, 996, 996, 996, 996, 996, 1129, 996, 996, 996, - /* 370 */ 996, 996, 996, 996, 996, 996, 996, 996, 996, 1114, - /* 380 */ 1112, 1111, 1110, 996, 1107, 996, 996, 996, 996, 996, - /* 390 */ 996, 996, 996, 996, 996, 996, 996, 996, 996, 996, - /* 400 */ 996, 996, 996, 996, 996, 996, 996, 996, 996, 996, - /* 410 */ 996, 996, 996, 996, 996, 996, 996, 996, 996, 996, - /* 420 */ 996, 996, 996, 996, 996, 996, 996, + /* 0 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 10 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 20 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 30 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 40 */ 999, 999, 999, 999, 1052, 999, 999, 999, 999, 999, + /* 50 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 60 */ 1050, 999, 1258, 999, 1164, 999, 999, 999, 999, 999, + /* 70 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 80 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 1052, + /* 90 */ 1269, 1269, 1269, 1050, 999, 999, 999, 999, 999, 1135, + /* 100 */ 999, 999, 999, 999, 999, 999, 999, 1333, 999, 1088, + /* 110 */ 1293, 999, 1285, 1261, 1275, 1262, 999, 1318, 1278, 999, + /* 120 */ 1169, 1166, 1166, 999, 999, 1052, 999, 999, 1052, 1052, + /* 130 */ 999, 1052, 999, 999, 999, 999, 999, 999, 999, 999, + /* 140 */ 999, 999, 999, 999, 999, 999, 999, 1050, 999, 999, + /* 150 */ 1050, 999, 1300, 1298, 999, 1300, 1298, 999, 999, 1312, + /* 160 */ 1308, 1291, 1289, 1275, 999, 999, 999, 1336, 1324, 1320, + /* 170 */ 999, 999, 1298, 999, 999, 1298, 999, 1177, 999, 999, + /* 180 */ 1050, 999, 1050, 999, 1104, 999, 1050, 999, 1138, 1138, + /* 190 */ 1053, 1004, 999, 999, 999, 999, 999, 999, 999, 999, + /* 200 */ 999, 999, 1230, 1311, 1310, 1229, 1235, 1234, 1233, 999, + /* 210 */ 999, 999, 1224, 1225, 1223, 1222, 999, 999, 999, 999, + /* 220 */ 999, 999, 999, 999, 999, 999, 999, 1259, 999, 1321, + /* 230 */ 1325, 999, 999, 999, 1209, 999, 999, 999, 999, 999, + /* 240 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 250 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 260 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 270 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 280 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 290 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 300 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 310 */ 999, 999, 1282, 1292, 999, 999, 999, 999, 999, 999, + /* 320 */ 999, 999, 999, 999, 1209, 999, 1309, 999, 1268, 1264, + /* 330 */ 999, 999, 1260, 999, 999, 1319, 999, 999, 999, 999, + /* 340 */ 999, 999, 999, 999, 1254, 999, 999, 999, 999, 999, + /* 350 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 360 */ 999, 1208, 999, 999, 999, 999, 999, 999, 999, 1132, + /* 370 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 380 */ 999, 999, 1117, 1115, 1114, 1113, 999, 1110, 999, 999, + /* 390 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 400 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 410 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, + /* 420 */ 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, }; /********** End of lemon-generated parsing tables *****************************/ @@ -1143,9 +1143,9 @@ static const char *const yyRuleName[] = { /* 169 */ "func_name_list ::= func_name", /* 170 */ "func_name_list ::= func_name_list NK_COMMA col_name", /* 171 */ "func_name ::= function_name", - /* 172 */ "cmd ::= CREATE SMA INDEX index_name ON table_name index_options", - /* 173 */ "cmd ::= CREATE FULLTEXT INDEX index_name ON table_name NK_LP col_name_list NK_RP", - /* 174 */ "cmd ::= DROP INDEX index_name ON table_name", + /* 172 */ "cmd ::= CREATE SMA INDEX not_exists_opt index_name ON table_name index_options", + /* 173 */ "cmd ::= CREATE FULLTEXT INDEX not_exists_opt index_name ON table_name NK_LP col_name_list NK_RP", + /* 174 */ "cmd ::= DROP INDEX exists_opt index_name ON table_name", /* 175 */ "index_options ::=", /* 176 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt", /* 177 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt", @@ -1263,7 +1263,7 @@ static const char *const yyRuleName[] = { /* 289 */ "partition_by_clause_opt ::=", /* 290 */ "partition_by_clause_opt ::= PARTITION BY expression_list", /* 291 */ "twindow_clause_opt ::=", - /* 292 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP", + /* 292 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", /* 293 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP", /* 294 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", /* 295 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", @@ -1325,10 +1325,10 @@ static int yyGrowStack(yyParser *p){ newSize = p->yystksz*2 + 100; idx = p->yytos ? (int)(p->yytos - p->yystack) : 0; if( p->yystack==&p->yystk0 ){ - pNew = malloc(newSize*sizeof(pNew[0])); + pNew = taosMemoryMalloc(newSize*sizeof(pNew[0])); if( pNew ) pNew[0] = p->yystk0; }else{ - pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); + pNew = taosMemoryRealloc(p->yystack, newSize*sizeof(pNew[0])); } if( pNew ){ p->yystack = pNew; @@ -1610,7 +1610,7 @@ void ParseFinalize(void *p){ yyParser *pParser = (yyParser*)p; while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); #if YYSTACKDEPTH<=0 - if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack); + if( pParser->yystack!=&pParser->yystk0 ) taosMemoryFree(pParser->yystack); #endif } @@ -2043,9 +2043,9 @@ static const struct { { 195, -1 }, /* (169) func_name_list ::= func_name */ { 195, -3 }, /* (170) func_name_list ::= func_name_list NK_COMMA col_name */ { 202, -1 }, /* (171) func_name ::= function_name */ - { 163, -7 }, /* (172) cmd ::= CREATE SMA INDEX index_name ON table_name index_options */ - { 163, -9 }, /* (173) cmd ::= CREATE FULLTEXT INDEX index_name ON table_name NK_LP col_name_list NK_RP */ - { 163, -5 }, /* (174) cmd ::= DROP INDEX index_name ON table_name */ + { 163, -8 }, /* (172) cmd ::= CREATE SMA INDEX not_exists_opt index_name ON table_name index_options */ + { 163, -10 }, /* (173) cmd ::= CREATE FULLTEXT INDEX not_exists_opt index_name ON table_name NK_LP col_name_list NK_RP */ + { 163, -6 }, /* (174) cmd ::= DROP INDEX exists_opt index_name ON table_name */ { 205, 0 }, /* (175) index_options ::= */ { 205, -9 }, /* (176) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ { 205, -11 }, /* (177) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt */ @@ -2163,7 +2163,7 @@ static const struct { { 240, 0 }, /* (289) partition_by_clause_opt ::= */ { 240, -3 }, /* (290) partition_by_clause_opt ::= PARTITION BY expression_list */ { 241, 0 }, /* (291) twindow_clause_opt ::= */ - { 241, -6 }, /* (292) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ + { 241, -6 }, /* (292) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ { 241, -4 }, /* (293) twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ { 241, -6 }, /* (294) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ { 241, -8 }, /* (295) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ @@ -2841,14 +2841,14 @@ static YYACTIONTYPE yy_reduce( { yylhsminor.yy140 = createFunctionNode(pCxt, &yymsp[0].minor.yy149, NULL); } yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 172: /* cmd ::= CREATE SMA INDEX index_name ON table_name index_options */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, &yymsp[-3].minor.yy149, &yymsp[-1].minor.yy149, NULL, yymsp[0].minor.yy140); } + case 172: /* cmd ::= CREATE SMA INDEX not_exists_opt index_name ON table_name index_options */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy497, &yymsp[-3].minor.yy149, &yymsp[-1].minor.yy149, NULL, yymsp[0].minor.yy140); } break; - case 173: /* cmd ::= CREATE FULLTEXT INDEX index_name ON table_name NK_LP col_name_list NK_RP */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, &yymsp[-5].minor.yy149, &yymsp[-3].minor.yy149, yymsp[-1].minor.yy136, NULL); } + case 173: /* cmd ::= CREATE FULLTEXT INDEX not_exists_opt index_name ON table_name NK_LP col_name_list NK_RP */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, yymsp[-6].minor.yy497, &yymsp[-5].minor.yy149, &yymsp[-3].minor.yy149, yymsp[-1].minor.yy136, NULL); } break; - case 174: /* cmd ::= DROP INDEX index_name ON table_name */ -{ pCxt->pRootNode = createDropIndexStmt(pCxt, &yymsp[-2].minor.yy149, &yymsp[0].minor.yy149); } + case 174: /* cmd ::= DROP INDEX exists_opt index_name ON table_name */ +{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-3].minor.yy497, &yymsp[-2].minor.yy149, &yymsp[0].minor.yy149); } break; case 176: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ { yymsp[-8].minor.yy140 = createIndexOption(pCxt, yymsp[-6].minor.yy136, releaseRawExprNode(pCxt, yymsp[-2].minor.yy140), NULL, yymsp[0].minor.yy140); } @@ -3230,8 +3230,8 @@ static YYACTIONTYPE yy_reduce( case 317: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==317); { yymsp[-2].minor.yy136 = yymsp[0].minor.yy136; } break; - case 292: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ -{ yymsp[-5].minor.yy140 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy140), &yymsp[-1].minor.yy0); } + case 292: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ +{ yymsp[-5].minor.yy140 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy140), releaseRawExprNode(pCxt, yymsp[-1].minor.yy140)); } break; case 293: /* twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ { yymsp[-3].minor.yy140 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy140)); } diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 9abaf5b14f49acc4e81eebab8f00841b2ab07261..012af26c17ce190c79c969960d6206a12de28f3b 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -65,7 +65,7 @@ private: friend class MockCatalogServiceImpl; static std::unique_ptr createTableBuilder(int8_t tableType, int32_t numOfColumns, int32_t numOfTags) { - STableMeta* meta = (STableMeta*)std::calloc(1, sizeof(STableMeta) + sizeof(SSchema) * (numOfColumns + numOfTags)); + STableMeta* meta = (STableMeta*)taosMemoryCalloc(1, sizeof(STableMeta) + sizeof(SSchema) * (numOfColumns + numOfTags)); if (nullptr == meta) { throw std::bad_alloc(); } @@ -120,23 +120,15 @@ public: } int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const { - // todo - vgInfo->vgId = 1; - addEpIntoEpSet(&vgInfo->epSet, "node1", 6030); - return 0; + char db[TSDB_DB_NAME_LEN] = {0}; + tNameGetDbName(pTableName, db); + return copyTableVgroup(db, tNameGetTableName(pTableName), vgInfo); } - int32_t catalogGetTableDistVgInfo(const SName* pTableName, SArray** pVgList) const { - SVgroupInfo info = {0}; - info.vgId = 1; - addEpIntoEpSet(&info.epSet, "node1", 6030); - - info.hashBegin = 0; - info.hashEnd = 1; - *pVgList = taosArrayInit(4, sizeof(SVgroupInfo)); - - taosArrayPush(*pVgList, &info); - return 0; + int32_t catalogGetTableDistVgInfo(const SName* pTableName, SArray** vgList) const { + char db[TSDB_DB_NAME_LEN] = {0}; + tNameGetDbName(pTableName, db); + return copyTableVgroup(db, tNameGetTableName(pTableName), vgList); } TableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags) { @@ -285,7 +277,7 @@ private: return TSDB_CODE_TSC_INVALID_TABLE_NAME; } int32_t len = sizeof(STableMeta) + sizeof(SSchema) * (src->tableInfo.numOfTags + src->tableInfo.numOfColumns); - dst->reset((STableMeta*)std::calloc(1, len)); + dst->reset((STableMeta*)taosMemoryCalloc(1, len)); if (!dst) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -293,6 +285,27 @@ private: return TSDB_CODE_SUCCESS; } + int32_t copyTableVgroup(const std::string& db, const std::string& tbname, SVgroupInfo* vg) const { + std::shared_ptr table = getTableMeta(db, tbname); + if (table->vgs.empty()) { + return TSDB_CODE_SUCCESS; + } + memcpy(vg, &(table->vgs[0]), sizeof(SVgroupInfo)); + return TSDB_CODE_SUCCESS; + } + + int32_t copyTableVgroup(const std::string& db, const std::string& tbname, SArray** vgList) const { + std::shared_ptr table = getTableMeta(db, tbname); + if (table->vgs.empty()) { + return TSDB_CODE_SUCCESS; + } + *vgList = taosArrayInit(table->vgs.size(), sizeof(SVgroupInfo)); + for (const SVgroupInfo& vg : table->vgs) { + taosArrayPush(*vgList, &vg); + } + return TSDB_CODE_SUCCESS; + } + uint64_t id_; std::unique_ptr builder_; DbMetaCache meta_; diff --git a/source/libs/parser/test/mockCatalogService.h b/source/libs/parser/test/mockCatalogService.h index a722f0b821255d9564f298842aa6d92afd37f0ae..887979f11e47af79a58ba7d94be4a6e46fbc17c2 100644 --- a/source/libs/parser/test/mockCatalogService.h +++ b/source/libs/parser/test/mockCatalogService.h @@ -44,7 +44,7 @@ public: struct MockTableMeta { ~MockTableMeta() { - free(schema); + taosMemoryFree(schema); } STableMeta* schema; diff --git a/source/libs/parser/test/parserAstTest.cpp b/source/libs/parser/test/parserAstTest.cpp index 8079e4f3bdd0a95767b05ea6e6b03e379956b825..1813ad0827fc47207224de624dfaaf1468cfecc8 100644 --- a/source/libs/parser/test/parserAstTest.cpp +++ b/source/libs/parser/test/parserAstTest.cpp @@ -44,6 +44,9 @@ protected: query_ = nullptr; bool res = runImpl(parseCode, translateCode); qDestroyQuery(query_); + if (!res) { + dump(); + } return res; } @@ -53,26 +56,40 @@ private: bool runImpl(int32_t parseCode, int32_t translateCode) { int32_t code = doParse(&cxt_, &query_); if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] parser code:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl; - return (TSDB_CODE_SUCCESS != parseCode); + parseErrStr_ = string("code:") + tstrerror(code) + string(", msg:") + errMagBuf_; + return (terrno == parseCode); } if (TSDB_CODE_SUCCESS != parseCode) { return false; } - string parserStr = toString(query_->pRoot); + parsedAstStr_ = toString(query_->pRoot); code = doTranslate(&cxt_, query_); if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] translate code:" << code << ", " << translateCode << ", msg:" << errMagBuf_ << endl; - return (code == translateCode); + translateErrStr_ = string("code:") + tstrerror(code) + string(", msg:") + errMagBuf_; + return (terrno == translateCode); } - cout << "input sql : [" << cxt_.pSql << "]" << endl; - cout << "parser output: " << endl; - cout << parserStr << endl; - cout << "translate output: " << endl; - cout << toString(query_->pRoot) << endl; + translatedAstStr_ = toString(query_->pRoot); return (TSDB_CODE_SUCCESS == translateCode); } + void dump() { + cout << "input sql : [" << cxt_.pSql << "]" << endl; + if (!parseErrStr_.empty()) { + cout << "parse error: " << parseErrStr_ << endl; + } + if (!parsedAstStr_.empty()) { + cout << "parse output: " << endl; + cout << parsedAstStr_ << endl; + } + if (!translateErrStr_.empty()) { + cout << "translate error: " << translateErrStr_ << endl; + } + if (!translatedAstStr_.empty()) { + cout << "translate output: " << endl; + cout << translatedAstStr_ << endl; + } + } + string toString(const SNode* pRoot, bool format = false) { char* pStr = NULL; int32_t len = 0; @@ -82,7 +99,7 @@ private: throw "nodesNodeToString failed!"; } string str(pStr); - tfree(pStr); + taosMemoryFreeClear(pStr); return str; } @@ -91,6 +108,10 @@ private: memset(errMagBuf_, 0, max_err_len); cxt_.pMsg = errMagBuf_; cxt_.msgLen = max_err_len; + parseErrStr_.clear(); + parsedAstStr_.clear(); + translateErrStr_.clear(); + translatedAstStr_.clear(); } string acctId_; @@ -99,8 +120,50 @@ private: string sqlBuf_; SParseContext cxt_; SQuery* query_; + string parseErrStr_; + string parsedAstStr_; + string translateErrStr_; + string translatedAstStr_; }; +TEST_F(ParserTest, createAccount) { + setDatabase("root", "test"); + + bind("create account ac_wxy pass '123456'"); + ASSERT_TRUE(run(TSDB_CODE_PAR_EXPRIE_STATEMENT)); +} + +TEST_F(ParserTest, alterAccount) { + setDatabase("root", "test"); + + bind("alter account ac_wxy pass '123456'"); + ASSERT_TRUE(run(TSDB_CODE_PAR_EXPRIE_STATEMENT)); +} + +TEST_F(ParserTest, createUser) { + setDatabase("root", "test"); + + bind("create user wxy pass '123456'"); + ASSERT_TRUE(run()); +} + +TEST_F(ParserTest, alterUser) { + setDatabase("root", "test"); + + bind("alter user wxy pass '123456'"); + ASSERT_TRUE(run()); + + bind("alter user wxy privilege 'write'"); + ASSERT_TRUE(run()); +} + +TEST_F(ParserTest, dropUser) { + setDatabase("root", "test"); + + bind("drop user wxy"); + ASSERT_TRUE(run()); +} + TEST_F(ParserTest, selectSimple) { setDatabase("root", "test"); @@ -295,13 +358,6 @@ TEST_F(ParserTest, selectSemanticError) { ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION)); } -TEST_F(ParserTest, createUser) { - setDatabase("root", "test"); - - bind("create user wxy pass '123456'"); - ASSERT_TRUE(run()); -} - TEST_F(ParserTest, showUsers) { setDatabase("root", "test"); @@ -309,13 +365,6 @@ TEST_F(ParserTest, showUsers) { ASSERT_TRUE(run()); } -TEST_F(ParserTest, alterAccount) { - setDatabase("root", "test"); - - bind("alter account ac_wxy pass '123456'"); - ASSERT_TRUE(run(TSDB_CODE_PAR_EXPRIE_STATEMENT)); -} - TEST_F(ParserTest, createDnode) { setDatabase("root", "test"); diff --git a/source/libs/parser/test/parserInsertTest.cpp b/source/libs/parser/test/parserInsertTest.cpp index cf40464d2b5d3393f65502142d7bd065197a2b0f..90e2ba3db2d18e8552078a5647c75525131b8211 100644 --- a/source/libs/parser/test/parserInsertTest.cpp +++ b/source/libs/parser/test/parserInsertTest.cpp @@ -72,7 +72,7 @@ protected: SSubmitBlk* blk = (SSubmitBlk*)(submit + 1); for (int32_t i = 0; i < numOfBlocks; ++i) { cout << "Block:" << i << endl; - cout << "\tuid:" << be64toh(blk->uid) << ", tid:" << ntohl(blk->tid) << ", padding:" << ntohl(blk->padding) << ", sversion:" << ntohl(blk->sversion) + cout << "\tuid:" << be64toh(blk->uid) << ", tid:" << be64toh(blk->suid) << ", padding:" << ntohl(blk->padding) << ", sversion:" << ntohl(blk->sversion) << ", dataLen:" << ntohl(blk->dataLen) << ", schemaLen:" << ntohl(blk->schemaLen) << ", numOfRows:" << ntohs(blk->numOfRows) << endl; blk = (SSubmitBlk*)(blk->data + ntohl(blk->dataLen)); } diff --git a/source/libs/planner/inc/planInt.h b/source/libs/planner/inc/planInt.h index be271cf05775ab7bdac5fd393ada78151621c5f6..42449d63d6c2d79ffe586f2d5ee1d96567d3bc85 100644 --- a/source/libs/planner/inc/planInt.h +++ b/source/libs/planner/inc/planInt.h @@ -56,9 +56,10 @@ extern "C" { #define planTrace(param, ...) qTrace("PLAN: " param, __VA_ARGS__) int32_t createLogicPlan(SPlanContext* pCxt, SLogicNode** pLogicNode); -int32_t optimize(SPlanContext* pCxt, SLogicNode* pLogicNode); -int32_t applySplitRule(SSubLogicPlan* pSubplan); -int32_t createPhysiPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SQueryPlan** pPlan, SArray* pExecNodeList); +int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode); +int32_t splitLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SLogicSubplan** pLogicSubplan); +int32_t scaleOutLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SQueryLogicPlan** pLogicPlan); +int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan, SArray* pExecNodeList); #ifdef __cplusplus } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 881db88503f26c7902d665d7d12d0d435a3cc085..84fa52a070e01e7cf203555d5a718b3bf20a9bd8 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -411,6 +411,38 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, return code; } +static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SWindowLogicNode* pWindow, SLogicNode** pLogicNode) { + int32_t code = nodesCollectFuncs(pSelect, fmIsAggFunc, &pWindow->pFuncs); + + if (TSDB_CODE_SUCCESS == code) { + code = rewriteExpr(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW); + } + + if (TSDB_CODE_SUCCESS == code) { + code = createColumnByRewriteExps(pCxt, pWindow->pFuncs, &pWindow->node.pTargets); + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pWindow; + } else { + nodesDestroyNode(pWindow); + } + + return code; +} + +static int32_t createWindowLogicNodeBySession(SLogicPlanContext* pCxt, SSessionWindowNode* pSession, SSelectStmt* pSelect, SLogicNode** pLogicNode) { + SWindowLogicNode* pWindow = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_WINDOW); + if (NULL == pWindow) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pWindow->winType = WINDOW_TYPE_SESSION; + pWindow->sessionGap = ((SValueNode*)pSession->pGap)->datum.i; + + return createWindowLogicNodeFinalize(pCxt, pSelect, pWindow, pLogicNode); +} + static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SIntervalWindowNode* pInterval, SSelectStmt* pSelect, SLogicNode** pLogicNode) { SWindowLogicNode* pWindow = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_WINDOW); if (NULL == pWindow) { @@ -424,34 +456,15 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva pWindow->sliding = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->datum.i : pWindow->interval); pWindow->slidingUnit = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : pWindow->intervalUnit); - int32_t code = TSDB_CODE_SUCCESS; - if (NULL != pInterval->pFill) { pWindow->pFill = nodesCloneNode(pInterval->pFill); if (NULL == pWindow->pFill) { - code = TSDB_CODE_OUT_OF_MEMORY; + nodesDestroyNode(pWindow); + return TSDB_CODE_OUT_OF_MEMORY; } } - if (TSDB_CODE_SUCCESS == code) { - code = nodesCollectFuncs(pSelect, fmIsAggFunc, &pWindow->pFuncs); - } - - if (TSDB_CODE_SUCCESS == code) { - code = rewriteExpr(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW); - } - - if (TSDB_CODE_SUCCESS == code) { - code = createColumnByRewriteExps(pCxt, pWindow->pFuncs, &pWindow->node.pTargets); - } - - if (TSDB_CODE_SUCCESS == code) { - *pLogicNode = (SLogicNode*)pWindow; - } else { - nodesDestroyNode(pWindow); - } - - return code; + return createWindowLogicNodeFinalize(pCxt, pSelect, pWindow, pLogicNode); } static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { @@ -460,8 +473,10 @@ static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele } switch (nodeType(pSelect->pWindow)) { + case QUERY_NODE_SESSION_WINDOW: + return createWindowLogicNodeBySession(pCxt, (SSessionWindowNode*)pSelect->pWindow, pSelect, pLogicNode); case QUERY_NODE_INTERVAL_WINDOW: - return createWindowLogicNodeByInterval(pCxt, (SIntervalWindowNode*)pSelect->pWindow, pSelect, pLogicNode); + return createWindowLogicNodeByInterval(pCxt, (SIntervalWindowNode*)pSelect->pWindow, pSelect, pLogicNode); default: break; } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c new file mode 100644 index 0000000000000000000000000000000000000000..2a36e38ce1f7f84de807a4f2eeee1229fcdc41ec --- /dev/null +++ b/source/libs/planner/src/planOptimizer.c @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planInt.h" + +int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode) { + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index a33cb8725356b50924092f2a165a310c58f921ca..fb9a2fc5325f4ccf866723a945142261f516575d 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -28,7 +28,6 @@ typedef struct SPhysiPlanContext { int16_t nextDataBlockId; SArray* pLocationHelper; SArray* pExecNodeList; - int32_t subplanId; } SPhysiPlanContext; static int32_t getSlotKey(SNode* pNode, char* pKey) { @@ -124,35 +123,52 @@ static EDealRes doSetSlotId(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static SNode* setNodeSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, int16_t rightDataBlockId, SNode* pNode) { +static int32_t setNodeSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, int16_t rightDataBlockId, SNode* pNode, SNode** pOutput) { SNode* pRes = nodesCloneNode(pNode); - CHECK_ALLOC(pRes, NULL); - SSetSlotIdCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId), - .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId)) }; + if (NULL == pRes) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SSetSlotIdCxt cxt = { + .errCode = TSDB_CODE_SUCCESS, + .pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId), + .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId)) + }; nodesWalkNode(pRes, doSetSlotId, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyNode(pRes); - return NULL; + return cxt.errCode; } - return pRes; + + *pOutput = pRes; + return TSDB_CODE_SUCCESS; } -static SNodeList* setListSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, int16_t rightDataBlockId, SNodeList* pList) { +static int32_t setListSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, int16_t rightDataBlockId, SNodeList* pList, SNodeList** pOutput) { SNodeList* pRes = nodesCloneList(pList); - CHECK_ALLOC(pRes, NULL); - SSetSlotIdCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId), - .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId)) }; + if (NULL == pRes) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SSetSlotIdCxt cxt = { + .errCode = TSDB_CODE_SUCCESS, + .pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId), + .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId)) + }; nodesWalkList(pRes, doSetSlotId, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyList(pRes); - return NULL; + return cxt.errCode; } - return pRes; + *pOutput = pRes; + return TSDB_CODE_SUCCESS; } static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, ENodeType type) { SPhysiNode* pPhysiNode = (SPhysiNode*)nodesMakeNode(type); - CHECK_ALLOC(pPhysiNode, NULL); + if (NULL == pPhysiNode) { + return NULL; + } pPhysiNode->pOutputDataBlockDesc = nodesMakeNode(QUERY_NODE_DATABLOCK_DESC); if (NULL == pPhysiNode->pOutputDataBlockDesc) { nodesDestroyNode(pPhysiNode); @@ -165,8 +181,7 @@ static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, ENodeType type) { static int32_t setConditionsSlotId(SPhysiPlanContext* pCxt, const SLogicNode* pLogicNode, SPhysiNode* pPhysiNode) { if (NULL != pLogicNode->pConditions) { - pPhysiNode->pConditions = setNodeSlotId(pCxt, pPhysiNode->pOutputDataBlockDesc->dataBlockId, -1, pLogicNode->pConditions); - CHECK_ALLOC(pPhysiNode->pConditions, TSDB_CODE_OUT_OF_MEMORY); + return setNodeSlotId(pCxt, pPhysiNode->pOutputDataBlockDesc->dataBlockId, -1, pLogicNode->pConditions, &pPhysiNode->pConditions); } return TSDB_CODE_SUCCESS; } @@ -198,6 +213,33 @@ static SNodeptr createPrimaryKeyCol(SPhysiPlanContext* pCxt, uint64_t tableId) { return pCol; } +static int32_t colIdCompare(const void* pLeft, const void* pRight) { + SColumnNode* pLeftCol = *(SColumnNode**)pLeft; + SColumnNode* pRightCol = *(SColumnNode**)pRight; + return pLeftCol->colId > pRightCol->colId ? 1 : -1; +} + +static int32_t sortScanCols(SNodeList* pScanCols) { + SArray* pArray = taosArrayInit(LIST_LENGTH(pScanCols), POINTER_BYTES); + if (NULL == pArray) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SNode* pCol = NULL; + FOREACH(pCol, pScanCols) { + taosArrayPush(pArray, &pCol); + } + taosArraySort(pArray, colIdCompare); + + int32_t index = 0; + FOREACH(pCol, pScanCols) { + REPLACE_NODE(taosArrayGetP(pArray, index++)); + } + taosArrayDestroy(pArray); + + return TSDB_CODE_SUCCESS; +} + static int32_t createScanCols(SPhysiPlanContext* pCxt, SScanPhysiNode* pScanPhysiNode, SNodeList* pScanCols) { if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pScanPhysiNode) || QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN == nodeType(pScanPhysiNode)) { @@ -220,27 +262,38 @@ static int32_t createScanCols(SPhysiPlanContext* pCxt, SScanPhysiNode* pScanPhys CHECK_ALLOC(pScanPhysiNode->pScanCols, TSDB_CODE_OUT_OF_MEMORY); } + // return sortScanCols(pScanPhysiNode->pScanCols); return TSDB_CODE_SUCCESS; } -static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SScanPhysiNode* pScanPhysiNode) { - CHECK_CODE(createScanCols(pCxt, pScanPhysiNode, pScanLogicNode->pScanCols), TSDB_CODE_OUT_OF_MEMORY); - - // Data block describe also needs to be set without scanning column, such as SELECT COUNT(*) FROM t - CHECK_CODE(addDataBlockDesc(pCxt, pScanPhysiNode->pScanCols, pScanPhysiNode->node.pOutputDataBlockDesc), TSDB_CODE_OUT_OF_MEMORY); - - CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pScanLogicNode, (SPhysiNode*)pScanPhysiNode), TSDB_CODE_OUT_OF_MEMORY); - - CHECK_CODE(setSlotOutput(pCxt, pScanLogicNode->node.pTargets, pScanPhysiNode->node.pOutputDataBlockDesc), TSDB_CODE_OUT_OF_MEMORY); +static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SScanPhysiNode* pScanPhysiNode, SPhysiNode** pPhyNode) { + int32_t code = createScanCols(pCxt, pScanPhysiNode, pScanLogicNode->pScanCols); + if (TSDB_CODE_SUCCESS == code) { + // Data block describe also needs to be set without scanning column, such as SELECT COUNT(*) FROM t + code = addDataBlockDesc(pCxt, pScanPhysiNode->pScanCols, pScanPhysiNode->node.pOutputDataBlockDesc); + } + if (TSDB_CODE_SUCCESS == code) { + code = setConditionsSlotId(pCxt, (const SLogicNode*)pScanLogicNode, (SPhysiNode*)pScanPhysiNode); + } + if (TSDB_CODE_SUCCESS == code) { + code = setSlotOutput(pCxt, pScanLogicNode->node.pTargets, pScanPhysiNode->node.pOutputDataBlockDesc); + } + if (TSDB_CODE_SUCCESS == code) { + pScanPhysiNode->uid = pScanLogicNode->pMeta->uid; + pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType; + pScanPhysiNode->order = TSDB_ORDER_ASC; + pScanPhysiNode->count = 1; + pScanPhysiNode->reverse = 0; + memcpy(&pScanPhysiNode->tableName, &pScanLogicNode->tableName, sizeof(SName)); + } - pScanPhysiNode->uid = pScanLogicNode->pMeta->uid; - pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType; - pScanPhysiNode->order = TSDB_ORDER_ASC; - pScanPhysiNode->count = 1; - pScanPhysiNode->reverse = 0; - memcpy(&pScanPhysiNode->tableName, &pScanLogicNode->tableName, sizeof(SName)); + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pScanPhysiNode; + } else { + nodesDestroyNode(pScanPhysiNode); + } - return TSDB_CODE_SUCCESS; + return code; } static void vgroupInfoToNodeAddr(const SVgroupInfo* vg, SQueryNodeAddr* pNodeAddr) { @@ -248,30 +301,36 @@ static void vgroupInfoToNodeAddr(const SVgroupInfo* vg, SQueryNodeAddr* pNodeAdd pNodeAddr->epSet = vg->epSet; } -static SPhysiNode* createTagScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) { +static int32_t createTagScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN); - CHECK_ALLOC(pTagScan, NULL); - CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTagScan), (SPhysiNode*)pTagScan); - return (SPhysiNode*)pTagScan; + if (NULL == pTagScan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + return createScanPhysiNodeFinalize(pCxt, pScanLogicNode, (SScanPhysiNode*)pTagScan, pPhyNode); } -static SPhysiNode* createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode) { +static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); - CHECK_ALLOC(pTableScan, NULL); - CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTableScan), (SPhysiNode*)pTableScan); + if (NULL == pTableScan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pTableScan->scanFlag = pScanLogicNode->scanFlag; pTableScan->scanRange = pScanLogicNode->scanRange; vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode); pSubplan->execNodeStat.tableNum = pScanLogicNode->pVgroupList->vgroups[0].numOfTable; tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName); - return (SPhysiNode*)pTableScan; + + return createScanPhysiNodeFinalize(pCxt, pScanLogicNode, (SScanPhysiNode*)pTableScan, pPhyNode); } -static SPhysiNode* createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode) { +static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { SSystemTableScanPhysiNode* pScan = (SSystemTableScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN); - CHECK_ALLOC(pScan, NULL); - CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pScan), (SPhysiNode*)pScan); + if (NULL == pScan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_USER_TABLES)) { vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode); @@ -284,88 +343,104 @@ static SPhysiNode* createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubp } pScan->mgmtEpSet = pCxt->pPlanCxt->mgmtEpSet; tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName); - return (SPhysiNode*)pScan; + + return createScanPhysiNodeFinalize(pCxt, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); } -static SPhysiNode* createStreamScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode) { - SStreamScanPhysiNode* pTableScan = (SStreamScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); - CHECK_ALLOC(pTableScan, NULL); - CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTableScan), (SPhysiNode*)pTableScan); - return (SPhysiNode*)pTableScan; +static int32_t createStreamScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { + SStreamScanPhysiNode* pScan = (SStreamScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); + if (NULL == pScan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + return createScanPhysiNodeFinalize(pCxt, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); } -static SPhysiNode* createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode) { +static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { switch (pScanLogicNode->scanType) { case SCAN_TYPE_TAG: - return createTagScanPhysiNode(pCxt, pScanLogicNode); + return createTagScanPhysiNode(pCxt, pScanLogicNode, pPhyNode); case SCAN_TYPE_TABLE: - return createTableScanPhysiNode(pCxt, pSubplan, pScanLogicNode); + return createTableScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); case SCAN_TYPE_SYSTEM_TABLE: - return createSystemTableScanPhysiNode(pCxt, pSubplan, pScanLogicNode); + return createSystemTableScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); case SCAN_TYPE_STREAM: - return createStreamScanPhysiNode(pCxt, pSubplan, pScanLogicNode); + return createStreamScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); default: break; } - return NULL; + return TSDB_CODE_FAILED; } -static SNodeList* createJoinOutputCols(SPhysiPlanContext* pCxt, SDataBlockDescNode* pLeftDesc, SDataBlockDescNode* pRightDesc) { - SNodeList* pCols = nodesMakeList(); - CHECK_ALLOC(pCols, NULL); +static int32_t createColFromDataBlockDesc(SDataBlockDescNode* pDesc, SNodeList* pCols) { SNode* pNode; - FOREACH(pNode, pLeftDesc->pSlots) { + FOREACH(pNode, pDesc->pSlots) { SSlotDescNode* pSlot = (SSlotDescNode*)pNode; SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pCol) { - goto error; + return TSDB_CODE_OUT_OF_MEMORY; } pCol->node.resType = pSlot->dataType; - pCol->dataBlockId = pLeftDesc->dataBlockId; + pCol->dataBlockId = pDesc->dataBlockId; pCol->slotId = pSlot->slotId; pCol->colId = -1; - if (TSDB_CODE_SUCCESS != nodesListAppend(pCols, (SNode*)pCol)) { - goto error; + int32_t code = nodesListStrictAppend(pCols, pCol); + if (TSDB_CODE_SUCCESS != code) { + return code; } } - FOREACH(pNode, pRightDesc->pSlots) { - SSlotDescNode* pSlot = (SSlotDescNode*)pNode; - SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); - if (NULL == pCol) { - goto error; - } - pCol->node.resType = pSlot->dataType; - pCol->dataBlockId = pRightDesc->dataBlockId; - pCol->slotId = pSlot->slotId; - pCol->colId = -1; - if (TSDB_CODE_SUCCESS != nodesListAppend(pCols, (SNode*)pCol)) { - goto error; - } + return TSDB_CODE_SUCCESS; +} + +static int32_t createJoinOutputCols(SPhysiPlanContext* pCxt, SDataBlockDescNode* pLeftDesc, SDataBlockDescNode* pRightDesc, SNodeList** pList) { + SNodeList* pCols = nodesMakeList(); + if (NULL == pCols) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = createColFromDataBlockDesc(pLeftDesc, pCols); + if (TSDB_CODE_SUCCESS == code) { + code = createColFromDataBlockDesc(pRightDesc, pCols); + } + + if (TSDB_CODE_SUCCESS == code) { + *pList = pCols; + } else { + nodesDestroyList(pCols); } - return pCols; -error: - nodesDestroyList(pCols); - return NULL; + + return code; } -static SPhysiNode* createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SJoinLogicNode* pJoinLogicNode) { +static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SJoinLogicNode* pJoinLogicNode, SPhysiNode** pPhyNode) { SJoinPhysiNode* pJoin = (SJoinPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_JOIN); - CHECK_ALLOC(pJoin, NULL); + if (NULL == pJoin) { + return TSDB_CODE_OUT_OF_MEMORY; + } SDataBlockDescNode* pLeftDesc = ((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc; SDataBlockDescNode* pRightDesc = ((SPhysiNode*)nodesListGetNode(pChildren, 1))->pOutputDataBlockDesc; - pJoin->pOnConditions = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pOnConditions); - CHECK_ALLOC(pJoin->pOnConditions, (SPhysiNode*)pJoin); - - pJoin->pTargets = createJoinOutputCols(pCxt, pLeftDesc, pRightDesc); - CHECK_ALLOC(pJoin->pTargets, (SPhysiNode*)pJoin); - CHECK_CODE(addDataBlockDesc(pCxt, pJoin->pTargets, pJoin->node.pOutputDataBlockDesc), (SPhysiNode*)pJoin); - CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin), (SPhysiNode*)pJoin); + int32_t code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pOnConditions, &pJoin->pOnConditions); + if (TSDB_CODE_SUCCESS == code) { + code = createJoinOutputCols(pCxt, pLeftDesc, pRightDesc, &pJoin->pTargets); + } + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockDesc(pCxt, pJoin->pTargets, pJoin->node.pOutputDataBlockDesc); + } + if (TSDB_CODE_SUCCESS == code) { + code = setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin); + } + if (TSDB_CODE_SUCCESS == code) { + code = setSlotOutput(pCxt, pJoinLogicNode->node.pTargets, pJoin->node.pOutputDataBlockDesc); + } - CHECK_CODE(setSlotOutput(pCxt, pJoinLogicNode->node.pTargets, pJoin->node.pOutputDataBlockDesc), (SPhysiNode*)pJoin); + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pJoin; + } else { + nodesDestroyNode(pJoin); + } - return (SPhysiNode*)pJoin; + return code; } typedef struct SRewritePrecalcExprsCxt { @@ -451,76 +526,169 @@ static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SN return cxt.errCode; } -static SPhysiNode* createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SAggLogicNode* pAggLogicNode) { +static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SAggLogicNode* pAggLogicNode, SPhysiNode** pPhyNode) { SAggPhysiNode* pAgg = (SAggPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_AGG); - CHECK_ALLOC(pAgg, NULL); + if (NULL == pAgg) { + return TSDB_CODE_OUT_OF_MEMORY; + } SNodeList* pPrecalcExprs = NULL; SNodeList* pGroupKeys = NULL; SNodeList* pAggFuncs = NULL; - CHECK_CODE(rewritePrecalcExprs(pCxt, pAggLogicNode->pGroupKeys, &pPrecalcExprs, &pGroupKeys), (SPhysiNode*)pAgg); - CHECK_CODE(rewritePrecalcExprs(pCxt, pAggLogicNode->pAggFuncs, &pPrecalcExprs, &pAggFuncs), (SPhysiNode*)pAgg); + int32_t code = rewritePrecalcExprs(pCxt, pAggLogicNode->pGroupKeys, &pPrecalcExprs, &pGroupKeys); + if (TSDB_CODE_SUCCESS == code) { + code = rewritePrecalcExprs(pCxt, pAggLogicNode->pAggFuncs, &pPrecalcExprs, &pAggFuncs); + } SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); // push down expression to pOutputDataBlockDesc of child node - if (NULL != pPrecalcExprs) { - pAgg->pExprs = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs); - CHECK_ALLOC(pAgg->pExprs, (SPhysiNode*)pAgg); - CHECK_CODE(addDataBlockDesc(pCxt, pAgg->pExprs, pChildTupe), (SPhysiNode*)pAgg); + if (TSDB_CODE_SUCCESS == code && NULL != pPrecalcExprs) { + code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs, &pAgg->pExprs); + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockDesc(pCxt, pAgg->pExprs, pChildTupe); + } } - if (NULL != pGroupKeys) { - pAgg->pGroupKeys = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pGroupKeys); - CHECK_ALLOC(pAgg->pGroupKeys, (SPhysiNode*)pAgg); - CHECK_CODE(addDataBlockDesc(pCxt, pAgg->pGroupKeys, pAgg->node.pOutputDataBlockDesc), (SPhysiNode*)pAgg); + if (TSDB_CODE_SUCCESS == code && NULL != pGroupKeys) { + code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pGroupKeys, &pAgg->pGroupKeys); + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockDesc(pCxt, pAgg->pGroupKeys, pAgg->node.pOutputDataBlockDesc); + } } - if (NULL != pAggFuncs) { - pAgg->pAggFuncs = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pAggFuncs); - CHECK_ALLOC(pAgg->pAggFuncs, (SPhysiNode*)pAgg); - CHECK_CODE(addDataBlockDesc(pCxt, pAgg->pAggFuncs, pAgg->node.pOutputDataBlockDesc), (SPhysiNode*)pAgg); + if (TSDB_CODE_SUCCESS == code && NULL != pAggFuncs) { + code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pAggFuncs, &pAgg->pAggFuncs); + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockDesc(pCxt, pAgg->pAggFuncs, pAgg->node.pOutputDataBlockDesc); + } } - CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pAggLogicNode, (SPhysiNode*)pAgg), (SPhysiNode*)pAgg); + if (TSDB_CODE_SUCCESS == code) { + code = setConditionsSlotId(pCxt, (const SLogicNode*)pAggLogicNode, (SPhysiNode*)pAgg); + } + if (TSDB_CODE_SUCCESS == code) { + code = setSlotOutput(pCxt, pAggLogicNode->node.pTargets, pAgg->node.pOutputDataBlockDesc); + } - CHECK_CODE(setSlotOutput(pCxt, pAggLogicNode->node.pTargets, pAgg->node.pOutputDataBlockDesc), (SPhysiNode*)pAgg); + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pAgg; + } else { + nodesDestroyNode(pAgg); + } - return (SPhysiNode*)pAgg; + return code; } -static SPhysiNode* createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SProjectLogicNode* pProjectLogicNode) { +static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SProjectLogicNode* pProjectLogicNode, SPhysiNode** pPhyNode) { SProjectPhysiNode* pProject = (SProjectPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_PROJECT); - CHECK_ALLOC(pProject, NULL); + if (NULL == pProject) { + return TSDB_CODE_OUT_OF_MEMORY; + } - pProject->pProjections = setListSlotId(pCxt, ((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc->dataBlockId, -1, pProjectLogicNode->pProjections); - CHECK_ALLOC(pProject->pProjections, (SPhysiNode*)pProject); - CHECK_CODE(addDataBlockDesc(pCxt, pProject->pProjections, pProject->node.pOutputDataBlockDesc), (SPhysiNode*)pProject); + int32_t code = setListSlotId(pCxt, ((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc->dataBlockId, -1, pProjectLogicNode->pProjections, &pProject->pProjections); + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockDesc(pCxt, pProject->pProjections, pProject->node.pOutputDataBlockDesc); + } + if (TSDB_CODE_SUCCESS == code) { + code = setConditionsSlotId(pCxt, (const SLogicNode*)pProjectLogicNode, (SPhysiNode*)pProject); + } - CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pProjectLogicNode, (SPhysiNode*)pProject), (SPhysiNode*)pProject); + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pProject; + } else { + nodesDestroyNode(pProject); + } - return (SPhysiNode*)pProject; + return code; } -static SPhysiNode* createExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode) { - if (pCxt->pPlanCxt->streamQuery) { - SStreamScanPhysiNode* pScan = (SStreamScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); - CHECK_ALLOC(pScan, NULL); +static int32_t doCreateExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode, SPhysiNode** pPhyNode) { + SExchangePhysiNode* pExchange = (SExchangePhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_EXCHANGE); + if (NULL == pExchange) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pExchange->srcGroupId = pExchangeLogicNode->srcGroupId; + int32_t code = addDataBlockDesc(pCxt, pExchangeLogicNode->node.pTargets, pExchange->node.pOutputDataBlockDesc); + + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pExchange; + } else { + nodesDestroyNode(pExchange); + } + + return code; +} +static int32_t createStreamScanPhysiNodeByExchange(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode, SPhysiNode** pPhyNode) { + SStreamScanPhysiNode* pScan = (SStreamScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); + if (NULL == pScan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = addDataBlockDesc(pCxt, pExchangeLogicNode->node.pTargets, pScan->node.pOutputDataBlockDesc); + if (TSDB_CODE_SUCCESS == code) { pScan->pScanCols = nodesCloneList(pExchangeLogicNode->node.pTargets); - CHECK_ALLOC(pScan->pScanCols, (SPhysiNode*)pScan); - CHECK_CODE(addDataBlockDesc(pCxt, pExchangeLogicNode->node.pTargets, pScan->node.pOutputDataBlockDesc), (SPhysiNode*)pScan); - return (SPhysiNode*)pScan; + if (NULL == pScan->pScanCols) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pScan; + } else { + nodesDestroyNode(pScan); + } + + return code; +} + +static int32_t createExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode, SPhysiNode** pPhyNode) { + if (pCxt->pPlanCxt->streamQuery) { + return createStreamScanPhysiNodeByExchange(pCxt, pExchangeLogicNode, pPhyNode); + } else { + return doCreateExchangePhysiNode(pCxt, pExchangeLogicNode, pPhyNode); + } +} + +static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWinodwPhysiNode* pWindow, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { + SNodeList* pPrecalcExprs = NULL; + SNodeList* pFuncs = NULL; + int32_t code = rewritePrecalcExprs(pCxt, pWindowLogicNode->pFuncs, &pPrecalcExprs, &pFuncs); + + SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); + // push down expression to pOutputDataBlockDesc of child node + if (TSDB_CODE_SUCCESS == code && NULL != pPrecalcExprs) { + code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs, &pWindow->pExprs); + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockDesc(pCxt, pWindow->pExprs, pChildTupe); + } + } + + if (TSDB_CODE_SUCCESS == code && NULL != pFuncs) { + code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pFuncs, &pWindow->pFuncs); + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockDesc(pCxt, pWindow->pFuncs, pWindow->node.pOutputDataBlockDesc); + } + } + + if (TSDB_CODE_SUCCESS == code) { + code = setSlotOutput(pCxt, pWindowLogicNode->node.pTargets, pWindow->node.pOutputDataBlockDesc); + } + + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pWindow; } else { - SExchangePhysiNode* pExchange = (SExchangePhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_EXCHANGE); - CHECK_ALLOC(pExchange, NULL); - CHECK_CODE(addDataBlockDesc(pCxt, pExchangeLogicNode->node.pTargets, pExchange->node.pOutputDataBlockDesc), (SPhysiNode*)pExchange); - pExchange->srcGroupId = pExchangeLogicNode->srcGroupId; - return (SPhysiNode*)pExchange; + nodesDestroyNode(pWindow); } + + return code; } -static SPhysiNode* createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode) { +static int32_t createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { SIntervalPhysiNode* pInterval = (SIntervalPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_INTERVAL); - CHECK_ALLOC(pInterval, NULL); + if (NULL == pInterval) { + return TSDB_CODE_OUT_OF_MEMORY; + } pInterval->interval = pWindowLogicNode->interval; pInterval->offset = pWindowLogicNode->offset; @@ -529,163 +697,178 @@ static SPhysiNode* createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* p pInterval->slidingUnit = pWindowLogicNode->slidingUnit; pInterval->pFill = nodesCloneNode(pWindowLogicNode->pFill); - - SNodeList* pPrecalcExprs = NULL; - SNodeList* pFuncs = NULL; - CHECK_CODE(rewritePrecalcExprs(pCxt, pWindowLogicNode->pFuncs, &pPrecalcExprs, &pFuncs), (SPhysiNode*)pInterval); - - SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); - // push down expression to pOutputDataBlockDesc of child node - if (NULL != pPrecalcExprs) { - pInterval->pExprs = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs); - CHECK_ALLOC(pInterval->pExprs, (SPhysiNode*)pInterval); - CHECK_CODE(addDataBlockDesc(pCxt, pInterval->pExprs, pChildTupe), (SPhysiNode*)pInterval); + if (NULL != pWindowLogicNode->pFill && NULL == pInterval->pFill) { + nodesDestroyNode(pInterval); + return TSDB_CODE_OUT_OF_MEMORY; } - if (NULL != pFuncs) { - pInterval->pFuncs = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pFuncs); - CHECK_ALLOC(pInterval->pFuncs, (SPhysiNode*)pInterval); - CHECK_CODE(addDataBlockDesc(pCxt, pInterval->pFuncs, pInterval->node.pOutputDataBlockDesc), (SPhysiNode*)pInterval); + return createWindowPhysiNodeFinalize(pCxt, pChildren, &pInterval->window, pWindowLogicNode, pPhyNode); +} + +static int32_t createSessionWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { + SSessionWinodwPhysiNode* pSession = (SSessionWinodwPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW); + if (NULL == pSession) { + return TSDB_CODE_OUT_OF_MEMORY; } - CHECK_CODE(setSlotOutput(pCxt, pWindowLogicNode->node.pTargets, pInterval->node.pOutputDataBlockDesc), (SPhysiNode*)pInterval); + pSession->gap = pWindowLogicNode->sessionGap; - return (SPhysiNode*)pInterval; + return createWindowPhysiNodeFinalize(pCxt, pChildren, &pSession->window, pWindowLogicNode, pPhyNode); } -static SPhysiNode* createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode) { +static int32_t createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { switch (pWindowLogicNode->winType) { case WINDOW_TYPE_INTERVAL: - return createIntervalPhysiNode(pCxt, pChildren, pWindowLogicNode); + return createIntervalPhysiNode(pCxt, pChildren, pWindowLogicNode, pPhyNode); case WINDOW_TYPE_SESSION: + return createSessionWindowPhysiNode(pCxt, pChildren, pWindowLogicNode, pPhyNode); case WINDOW_TYPE_STATE: break; default: break; } - return NULL; + return TSDB_CODE_FAILED; } -static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SLogicNode* pLogicPlan) { - SNodeList* pChildren = nodesMakeList(); - CHECK_ALLOC(pChildren, NULL); - - SNode* pLogicChild; - FOREACH(pLogicChild, pLogicPlan->pChildren) { - if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pChildren, createPhysiNode(pCxt, pSubplan, (SLogicNode*)pLogicChild))) { - pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; - nodesDestroyList(pChildren); - return NULL; - } - } - - SPhysiNode* pPhyNode = NULL; - switch (nodeType(pLogicPlan)) { +static int32_t doCreatePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, SSubplan* pSubplan, SNodeList* pChildren, SPhysiNode** pPhyNode) { + switch (nodeType(pLogicNode)) { case QUERY_NODE_LOGIC_PLAN_SCAN: - pPhyNode = createScanPhysiNode(pCxt, pSubplan, (SScanLogicNode*)pLogicPlan); - break; + return createScanPhysiNode(pCxt, pSubplan, (SScanLogicNode*)pLogicNode, pPhyNode); case QUERY_NODE_LOGIC_PLAN_JOIN: - pPhyNode = createJoinPhysiNode(pCxt, pChildren, (SJoinLogicNode*)pLogicPlan); - break; + return createJoinPhysiNode(pCxt, pChildren, (SJoinLogicNode*)pLogicNode, pPhyNode); case QUERY_NODE_LOGIC_PLAN_AGG: - pPhyNode = createAggPhysiNode(pCxt, pChildren, (SAggLogicNode*)pLogicPlan); - break; + return createAggPhysiNode(pCxt, pChildren, (SAggLogicNode*)pLogicNode, pPhyNode); case QUERY_NODE_LOGIC_PLAN_PROJECT: - pPhyNode = createProjectPhysiNode(pCxt, pChildren, (SProjectLogicNode*)pLogicPlan); - break; + return createProjectPhysiNode(pCxt, pChildren, (SProjectLogicNode*)pLogicNode, pPhyNode); case QUERY_NODE_LOGIC_PLAN_EXCHANGE: - pPhyNode = createExchangePhysiNode(pCxt, (SExchangeLogicNode*)pLogicPlan); - break; + return createExchangePhysiNode(pCxt, (SExchangeLogicNode*)pLogicNode, pPhyNode); case QUERY_NODE_LOGIC_PLAN_WINDOW: - pPhyNode = createWindowPhysiNode(pCxt, pChildren, (SWindowLogicNode*)pLogicPlan); - break; + return createWindowPhysiNode(pCxt, pChildren, (SWindowLogicNode*)pLogicNode, pPhyNode); default: break; } - if (TSDB_CODE_SUCCESS != pCxt->errCode) { - nodesDestroyNode(pPhyNode); - return NULL; + + return TSDB_CODE_FAILED; +} + +static int32_t createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, SSubplan* pSubplan, SPhysiNode** pPhyNode) { + SNodeList* pChildren = nodesMakeList(); + if (NULL == pChildren) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = TSDB_CODE_SUCCESS; + + SNode* pLogicChild; + FOREACH(pLogicChild, pLogicNode->pChildren) { + SPhysiNode* pChild = NULL; + code = createPhysiNode(pCxt, (SLogicNode*)pLogicChild, pSubplan, &pChild); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListStrictAppend(pChildren, pChild); + } + } + + if (TSDB_CODE_SUCCESS == code) { + code = doCreatePhysiNode(pCxt, pLogicNode, pSubplan, pChildren, pPhyNode); } - pPhyNode->pChildren = pChildren; - SNode* pChild; - FOREACH(pChild, pPhyNode->pChildren) { - ((SPhysiNode*)pChild)->pParent = pPhyNode; + if (TSDB_CODE_SUCCESS == code) { + (*pPhyNode)->pChildren = pChildren; + SNode* pChild; + FOREACH(pChild, (*pPhyNode)->pChildren) { + ((SPhysiNode*)pChild)->pParent = (*pPhyNode); + } + } else { + nodesDestroyList(pChildren); } - return pPhyNode; + return code; } -static SDataSinkNode* createDataInserter(SPhysiPlanContext* pCxt, SVgDataBlocks* pBlocks) { +static int32_t createDataInserter(SPhysiPlanContext* pCxt, SVgDataBlocks* pBlocks, SDataSinkNode** pSink) { SDataInserterNode* pInserter = nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_INSERT); - CHECK_ALLOC(pInserter, NULL); + if (NULL == pInserter) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pInserter->numOfTables = pBlocks->numOfTables; pInserter->size = pBlocks->size; TSWAP(pInserter->pData, pBlocks->pData, char*); - return (SDataSinkNode*)pInserter; + + *pSink = (SDataSinkNode*)pInserter; + return TSDB_CODE_SUCCESS; } -static SDataSinkNode* createDataDispatcher(SPhysiPlanContext* pCxt, const SPhysiNode* pRoot) { +static int32_t createDataDispatcher(SPhysiPlanContext* pCxt, const SPhysiNode* pRoot, SDataSinkNode** pSink) { SDataDispatcherNode* pDispatcher = nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_DISPATCH); - CHECK_ALLOC(pDispatcher, NULL); + if (NULL == pDispatcher) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pDispatcher->sink.pInputDataBlockDesc = nodesCloneNode(pRoot->pOutputDataBlockDesc); - CHECK_ALLOC(pDispatcher->sink.pInputDataBlockDesc, (SDataSinkNode*)pDispatcher); - return (SDataSinkNode*)pDispatcher; + if (NULL == pDispatcher->sink.pInputDataBlockDesc) { + nodesDestroyNode(pDispatcher); + return TSDB_CODE_OUT_OF_MEMORY; + } + + *pSink = (SDataSinkNode*)pDispatcher; + return TSDB_CODE_SUCCESS; } -static SSubplan* makeSubplan(SPhysiPlanContext* pCxt, SSubLogicPlan* pLogicSubplan) { +static SSubplan* makeSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubplan) { SSubplan* pSubplan = nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN); - CHECK_ALLOC(pSubplan, NULL); + if (NULL == pSubplan) { + return NULL; + } pSubplan->id = pLogicSubplan->id; pSubplan->subplanType = pLogicSubplan->subplanType; pSubplan->level = pLogicSubplan->level; return pSubplan; } -static SSubplan* createPhysiSubplan(SPhysiPlanContext* pCxt, SSubLogicPlan* pLogicSubplan) { +static int32_t createPhysiSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SSubplan** pPhysiSubplan) { SSubplan* pSubplan = makeSubplan(pCxt, pLogicSubplan); - CHECK_ALLOC(pSubplan, NULL); + if (NULL == pSubplan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = TSDB_CODE_SUCCESS; + if (SUBPLAN_TYPE_MODIFY == pLogicSubplan->subplanType) { SVnodeModifLogicNode* pModif = (SVnodeModifLogicNode*)pLogicSubplan->pNode; - pSubplan->pDataSink = createDataInserter(pCxt, pModif->pVgDataBlocks); pSubplan->msgType = pModif->msgType; pSubplan->execNode.epSet = pModif->pVgDataBlocks->vg.epSet; taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode); + code = createDataInserter(pCxt, pModif->pVgDataBlocks, &pSubplan->pDataSink); } else { - pSubplan->pNode = createPhysiNode(pCxt, pSubplan, pLogicSubplan->pNode); - if (!pCxt->pPlanCxt->streamQuery && !pCxt->pPlanCxt->topicQuery) { - pSubplan->pDataSink = createDataDispatcher(pCxt, pSubplan->pNode); - } pSubplan->msgType = TDMT_VND_QUERY; + code = createPhysiNode(pCxt, pLogicSubplan->pNode, pSubplan, &pSubplan->pNode); + if (TSDB_CODE_SUCCESS == code && !pCxt->pPlanCxt->streamQuery && !pCxt->pPlanCxt->topicQuery) { + code = createDataDispatcher(pCxt, pSubplan->pNode, &pSubplan->pDataSink); + } } - return pSubplan; -} -static void doSetLogicNodeParent(SLogicNode* pNode, SLogicNode* pParent) { - pNode->pParent = pParent; - SNode* pChild; - FOREACH(pChild, pNode->pChildren) { - doSetLogicNodeParent((SLogicNode*)pChild, pNode); + if (TSDB_CODE_SUCCESS == code) { + *pPhysiSubplan = pSubplan; + } else { + nodesDestroyNode(pSubplan); } + + return code; } -static void setLogicNodeParent(SLogicNode* pNode) { - doSetLogicNodeParent(pNode, NULL); -} - -static int32_t splitLogicPlan(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, SSubLogicPlan** pSubLogicPlan) { - *pSubLogicPlan = (SSubLogicPlan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); - CHECK_ALLOC(*pSubLogicPlan, TSDB_CODE_OUT_OF_MEMORY); - (*pSubLogicPlan)->pNode = nodesCloneNode(pLogicNode); - if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIF == nodeType(pLogicNode)) { - (*pSubLogicPlan)->subplanType = SUBPLAN_TYPE_MODIFY; - TSWAP(((SVnodeModifLogicNode*)pLogicNode)->pDataBlocks, ((SVnodeModifLogicNode*)(*pSubLogicPlan)->pNode)->pDataBlocks, SArray*); - } else { - (*pSubLogicPlan)->subplanType = SUBPLAN_TYPE_SCAN; +static SQueryPlan* makeQueryPhysiPlan(SPhysiPlanContext* pCxt) { + SQueryPlan* pPlan = nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN); + if (NULL == pPlan) { + return NULL; + } + pPlan->pSubplans = nodesMakeList(); + if (NULL == pPlan->pSubplans) { + nodesDestroyNode(pPlan); + return NULL; } - (*pSubLogicPlan)->id.queryId = pCxt->pPlanCxt->queryId; - setLogicNodeParent((*pSubLogicPlan)->pNode); - return applySplitRule(*pSubLogicPlan); + pPlan->queryId = pCxt->pPlanCxt->queryId; + return pPlan; } static int32_t pushSubplan(SPhysiPlanContext* pCxt, SNodeptr pSubplan, int32_t level, SNodeList* pSubplans) { @@ -705,193 +888,65 @@ static int32_t pushSubplan(SPhysiPlanContext* pCxt, SNodeptr pSubplan, int32_t l return TSDB_CODE_SUCCESS; } -static SSubLogicPlan* singleCloneSubLogicPlan(SPhysiPlanContext* pCxt, SSubLogicPlan* pSrc, int32_t level) { - SSubLogicPlan* pDst = nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); - CHECK_ALLOC(pDst, NULL); - pDst->pNode = nodesCloneNode(pSrc->pNode); - if (NULL == pDst->pNode) { - nodesDestroyNode(pDst); - return NULL; - } - pDst->subplanType = pSrc->subplanType; - pDst->level = level; - pDst->id.queryId = pSrc->id.queryId; - pDst->id.groupId = pSrc->id.groupId; - pDst->id.subplanId = pCxt->subplanId++; - return pDst; -} +static int32_t buildPhysiPlan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SSubplan* pParent, SQueryPlan* pQueryPlan) { + SSubplan* pSubplan = NULL; + int32_t code = createPhysiSubplan(pCxt, pLogicSubplan, &pSubplan); -static int32_t scaleOutForModify(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t level, SNodeList* pGroup) { - SVnodeModifLogicNode* pNode = (SVnodeModifLogicNode*)pSubplan->pNode; - size_t numOfVgroups = taosArrayGetSize(pNode->pDataBlocks); - for (int32_t i = 0; i < numOfVgroups; ++i) { - SSubLogicPlan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level); - CHECK_ALLOC(pNewSubplan, TSDB_CODE_OUT_OF_MEMORY); - SVgDataBlocks* blocks = (SVgDataBlocks*)taosArrayGetP(pNode->pDataBlocks, i); - ((SVnodeModifLogicNode*)pNewSubplan->pNode)->pVgDataBlocks = blocks; - CHECK_CODE_EXT(nodesListAppend(pGroup, pNewSubplan)); - } - return TSDB_CODE_SUCCESS; -} - -static int32_t scaleOutForMerge(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t level, SNodeList* pGroup) { - return nodesListStrictAppend(pGroup, singleCloneSubLogicPlan(pCxt, pSubplan, level)); -} - -static int32_t doSetScanVgroup(SPhysiPlanContext* pCxt, SLogicNode* pNode, const SVgroupInfo* pVgroup, bool* pFound) { - if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { - SScanLogicNode* pScan = (SScanLogicNode*)pNode; - pScan->pVgroupList = calloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo)); - CHECK_ALLOC(pScan->pVgroupList, TSDB_CODE_OUT_OF_MEMORY); - memcpy(pScan->pVgroupList->vgroups, pVgroup, sizeof(SVgroupInfo)); - *pFound = true; - return TSDB_CODE_SUCCESS; - } - SNode* pChild = NULL; - FOREACH(pChild, pNode->pChildren) { - int32_t code = doSetScanVgroup(pCxt, (SLogicNode*)pChild, pVgroup, pFound); - if (TSDB_CODE_SUCCESS != code || *pFound) { - return code; - } - } - return TSDB_CODE_SUCCESS; -} - -static int32_t setScanVgroup(SPhysiPlanContext* pCxt, SLogicNode* pNode, const SVgroupInfo* pVgroup) { - bool found = false; - return doSetScanVgroup(pCxt, pNode, pVgroup, &found); -} - -static int32_t scaleOutForScan(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t level, SNodeList* pGroup) { - if (pSubplan->pVgroupList) { - for (int32_t i = 0; i < pSubplan->pVgroupList->numOfVgroups; ++i) { - SSubLogicPlan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level); - CHECK_ALLOC(pNewSubplan, TSDB_CODE_OUT_OF_MEMORY); - CHECK_CODE_EXT(setScanVgroup(pCxt, pNewSubplan->pNode, pSubplan->pVgroupList->vgroups + i)); - CHECK_CODE_EXT(nodesListAppend(pGroup, pNewSubplan)); - } - return TSDB_CODE_SUCCESS; - } else { - return scaleOutForMerge(pCxt, pSubplan, level, pGroup); + if (TSDB_CODE_SUCCESS == code) { + code = pushSubplan(pCxt, pSubplan, pLogicSubplan->level, pQueryPlan->pSubplans); + ++(pQueryPlan->numOfSubplans); } -} -static int32_t appendWithMakeList(SNodeList** pList, SNodeptr pNode) { - if (NULL == *pList) { - *pList = nodesMakeList(); - if (NULL == *pList) { - return TSDB_CODE_OUT_OF_MEMORY; + if (TSDB_CODE_SUCCESS == code && NULL != pParent) { + code = nodesListMakeAppend(&pParent->pChildren, pSubplan); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeAppend(&pSubplan->pParents, pParent); } } - return nodesListAppend(*pList, pNode); -} -static int32_t pushHierarchicalPlan(SPhysiPlanContext* pCxt, SNodeList* pParentsGroup, SNodeList* pCurrentGroup) { - bool topLevel = (0 == LIST_LENGTH(pParentsGroup)); - SNode* pChild = NULL; - FOREACH(pChild, pCurrentGroup) { - if (topLevel) { - CHECK_CODE_EXT(nodesListAppend(pParentsGroup, pChild)); - } else { - SNode* pParent = NULL; - FOREACH(pParent, pParentsGroup) { - CHECK_CODE_EXT(appendWithMakeList(&(((SSubLogicPlan*)pParent)->pChildren), pChild)); - CHECK_CODE_EXT(appendWithMakeList(&(((SSubLogicPlan*)pChild)->pParents), pParent)); + if (TSDB_CODE_SUCCESS == code) { + SNode* pChild = NULL; + FOREACH(pChild, pLogicSubplan->pChildren) { + code = buildPhysiPlan(pCxt, (SLogicSubplan*)pChild, pSubplan, pQueryPlan); + if (TSDB_CODE_SUCCESS != code) { + break; } } } - return TSDB_CODE_SUCCESS; -} -static int32_t doScaleOut(SPhysiPlanContext* pCxt, SSubLogicPlan* pSubplan, int32_t* pLevel, SNodeList* pParentsGroup) { - SNodeList* pCurrentGroup = nodesMakeList(); - CHECK_ALLOC(pCurrentGroup, TSDB_CODE_OUT_OF_MEMORY); - int32_t code = TSDB_CODE_SUCCESS; - switch (pSubplan->subplanType) { - case SUBPLAN_TYPE_MERGE: - code = scaleOutForMerge(pCxt, pSubplan, *pLevel, pCurrentGroup); - break; - case SUBPLAN_TYPE_SCAN: - code = scaleOutForScan(pCxt, pSubplan, *pLevel, pCurrentGroup); - break; - case SUBPLAN_TYPE_MODIFY: - code = scaleOutForModify(pCxt, pSubplan, *pLevel, pCurrentGroup); - break; - default: - break; - } if (TSDB_CODE_SUCCESS != code) { - return code; - } - - CHECK_CODE_EXT(pushHierarchicalPlan(pCxt, pParentsGroup, pCurrentGroup)); - ++(*pLevel); - SNode* pChild; - FOREACH(pChild, pSubplan->pChildren) { - CHECK_CODE_EXT(doScaleOut(pCxt, (SSubLogicPlan*)pChild, pLevel, pCurrentGroup)); + nodesDestroyNode(pSubplan); } - return TSDB_CODE_SUCCESS; + return code; } -static SQueryLogicPlan* makeQueryLogicPlan(SPhysiPlanContext* pCxt) { - SQueryLogicPlan* pLogicPlan = (SQueryLogicPlan*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN); - CHECK_ALLOC(pLogicPlan, NULL); - pLogicPlan->pTopSubplans = nodesMakeList(); - if (NULL == pLogicPlan->pTopSubplans) { - nodesDestroyNode(pLogicPlan); - return NULL; +static int32_t doCreatePhysiPlan(SPhysiPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPhysiPlan) { + SQueryPlan* pPlan = makeQueryPhysiPlan(pCxt); + if (NULL == pPlan) { + return TSDB_CODE_OUT_OF_MEMORY; } - return pLogicPlan; -} -static int32_t scaleOutLogicPlan(SPhysiPlanContext* pCxt, SSubLogicPlan* pRootSubLogicPlan, SQueryLogicPlan** pLogicPlan) { - *pLogicPlan = makeQueryLogicPlan(pCxt); - CHECK_ALLOC(*pLogicPlan, TSDB_CODE_OUT_OF_MEMORY); - return doScaleOut(pCxt, pRootSubLogicPlan, &((*pLogicPlan)->totalLevel), (*pLogicPlan)->pTopSubplans); -} - -static SQueryPlan* makeQueryPhysiPlan(SPhysiPlanContext* pCxt) { - SQueryPlan* pPlan = nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN); - CHECK_ALLOC(pPlan, NULL); - pPlan->pSubplans = nodesMakeList(); - if (NULL == pPlan->pSubplans) { - nodesDestroyNode(pPlan); - return NULL; - } - pPlan->queryId = pCxt->pPlanCxt->queryId; - return pPlan; -} + int32_t code = TSDB_CODE_SUCCESS; -static int32_t doBuildPhysiPlan(SPhysiPlanContext* pCxt, SSubLogicPlan* pLogicSubplan, SSubplan* pParent, SQueryPlan* pQueryPlan) { - SSubplan* pSubplan = createPhysiSubplan(pCxt, pLogicSubplan); - CHECK_ALLOC(pSubplan, TSDB_CODE_OUT_OF_MEMORY); - CHECK_CODE_EXT(pushSubplan(pCxt, pSubplan, pLogicSubplan->level, pQueryPlan->pSubplans)); - ++(pQueryPlan->numOfSubplans); - if (NULL != pParent) { - CHECK_CODE_EXT(appendWithMakeList(&pParent->pChildren, pSubplan)); - CHECK_CODE_EXT(appendWithMakeList(&pSubplan->pParents, pParent)); + SNode* pSubplan = NULL; + FOREACH(pSubplan, pLogicPlan->pTopSubplans) { + code = buildPhysiPlan(pCxt, (SLogicSubplan*)pSubplan, NULL, pPlan); + if (TSDB_CODE_SUCCESS != code) { + break; + } } - SNode* pChild = NULL; - FOREACH(pChild, pLogicSubplan->pChildren) { - CHECK_CODE_EXT(doBuildPhysiPlan(pCxt, (SSubLogicPlan*)pChild, pSubplan, pQueryPlan)); + if (TSDB_CODE_SUCCESS == code) { + *pPhysiPlan = pPlan; + } else { + nodesDestroyNode(pPlan); } - return TSDB_CODE_SUCCESS; -} - -static int32_t buildPhysiPlan(SPhysiPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan) { - *pPlan = makeQueryPhysiPlan(pCxt); - CHECK_ALLOC(*pPlan, TSDB_CODE_OUT_OF_MEMORY); - SNode* pSubplan = NULL; - FOREACH(pSubplan, pLogicPlan->pTopSubplans) { - CHECK_CODE_EXT(doBuildPhysiPlan(pCxt, (SSubLogicPlan*)pSubplan, NULL, *pPlan)); - } - return TSDB_CODE_SUCCESS; + return code; } -int32_t createPhysiPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SQueryPlan** pPlan, SArray* pExecNodeList) { +int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan, SArray* pExecNodeList) { SPhysiPlanContext cxt = { .pPlanCxt = pCxt, .errCode = TSDB_CODE_SUCCESS, @@ -902,16 +957,5 @@ int32_t createPhysiPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SQueryPlan** if (NULL == cxt.pLocationHelper) { return TSDB_CODE_OUT_OF_MEMORY; } - SQueryLogicPlan* pLogicPlan = NULL; - SSubLogicPlan* pSubLogicPlan = NULL; - int32_t code = splitLogicPlan(&cxt, pLogicNode, &pSubLogicPlan); - if (TSDB_CODE_SUCCESS == code) { - code = scaleOutLogicPlan(&cxt, pSubLogicPlan, &pLogicPlan); - } - if (TSDB_CODE_SUCCESS == code) { - code = buildPhysiPlan(&cxt, pLogicPlan, pPlan); - } - nodesDestroyNode(pSubLogicPlan); - nodesDestroyNode(pLogicPlan); - return code; + return doCreatePhysiPlan(&cxt, pLogicPlan, pPlan); } diff --git a/source/libs/planner/src/planScaleOut.c b/source/libs/planner/src/planScaleOut.c new file mode 100644 index 0000000000000000000000000000000000000000..ca6c7a2577e3439bcb50c907ea126ea6fb265b46 --- /dev/null +++ b/source/libs/planner/src/planScaleOut.c @@ -0,0 +1,203 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planInt.h" + +typedef struct SScaleOutContext { + SPlanContext* pPlanCxt; + int32_t subplanId; +} SScaleOutContext; + +static SLogicSubplan* singleCloneSubLogicPlan(SScaleOutContext* pCxt, SLogicSubplan* pSrc, int32_t level) { + SLogicSubplan* pDst = nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); + if (NULL == pDst) { + return NULL; + } + pDst->pNode = nodesCloneNode(pSrc->pNode); + if (NULL == pDst->pNode) { + nodesDestroyNode(pDst); + return NULL; + } + pDst->subplanType = pSrc->subplanType; + pDst->level = level; + pDst->id.queryId = pSrc->id.queryId; + pDst->id.groupId = pSrc->id.groupId; + pDst->id.subplanId = pCxt->subplanId++; + return pDst; +} + +static int32_t scaleOutForModify(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) { + SVnodeModifLogicNode* pNode = (SVnodeModifLogicNode*)pSubplan->pNode; + size_t numOfVgroups = taosArrayGetSize(pNode->pDataBlocks); + for (int32_t i = 0; i < numOfVgroups; ++i) { + SLogicSubplan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level); + if (NULL == pNewSubplan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + ((SVnodeModifLogicNode*)pNewSubplan->pNode)->pVgDataBlocks = (SVgDataBlocks*)taosArrayGetP(pNode->pDataBlocks, i); + if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pGroup, pNewSubplan)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t scaleOutForMerge(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) { + return nodesListStrictAppend(pGroup, singleCloneSubLogicPlan(pCxt, pSubplan, level)); +} + +static int32_t doSetScanVgroup(SLogicNode* pNode, const SVgroupInfo* pVgroup, bool* pFound) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { + SScanLogicNode* pScan = (SScanLogicNode*)pNode; + pScan->pVgroupList = taosMemoryCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo)); + if (NULL == pScan->pVgroupList) { + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(pScan->pVgroupList->vgroups, pVgroup, sizeof(SVgroupInfo)); + *pFound = true; + return TSDB_CODE_SUCCESS; + } + SNode* pChild = NULL; + FOREACH(pChild, pNode->pChildren) { + int32_t code = doSetScanVgroup((SLogicNode*)pChild, pVgroup, pFound); + if (TSDB_CODE_SUCCESS != code || *pFound) { + return code; + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t setScanVgroup(SLogicNode* pNode, const SVgroupInfo* pVgroup) { + bool found = false; + return doSetScanVgroup(pNode, pVgroup, &found); +} + +static int32_t scaleOutForScan(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) { + if (pSubplan->pVgroupList && !pCxt->pPlanCxt->streamQuery) { + int32_t code = TSDB_CODE_SUCCESS; + for (int32_t i = 0; i < pSubplan->pVgroupList->numOfVgroups; ++i) { + SLogicSubplan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level); + if (NULL == pNewSubplan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + code = setScanVgroup(pNewSubplan->pNode, pSubplan->pVgroupList->vgroups + i); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListStrictAppend(pGroup, pNewSubplan); + } + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + return code; + } else { + return scaleOutForMerge(pCxt, pSubplan, level, pGroup); + } +} + +static int32_t pushHierarchicalPlan(SNodeList* pParentsGroup, SNodeList* pCurrentGroup) { + int32_t code = TSDB_CODE_SUCCESS; + bool topLevel = (0 == LIST_LENGTH(pParentsGroup)); + SNode* pChild = NULL; + FOREACH(pChild, pCurrentGroup) { + if (topLevel) { + code = nodesListAppend(pParentsGroup, pChild); + } else { + SNode* pParent = NULL; + FOREACH(pParent, pParentsGroup) { + code = nodesListMakeAppend(&(((SLogicSubplan*)pParent)->pChildren), pChild); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeAppend(&(((SLogicSubplan*)pChild)->pParents), pParent); + } + } + } + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + return code; +} + +static int32_t doScaleOut(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t* pLevel, SNodeList* pParentsGroup) { + SNodeList* pCurrentGroup = nodesMakeList(); + if (NULL == pCurrentGroup) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = TSDB_CODE_SUCCESS; + switch (pSubplan->subplanType) { + case SUBPLAN_TYPE_MERGE: + code = scaleOutForMerge(pCxt, pSubplan, *pLevel, pCurrentGroup); + break; + case SUBPLAN_TYPE_SCAN: + code = scaleOutForScan(pCxt, pSubplan, *pLevel, pCurrentGroup); + break; + case SUBPLAN_TYPE_MODIFY: + code = scaleOutForModify(pCxt, pSubplan, *pLevel, pCurrentGroup); + break; + default: + break; + } + + if (TSDB_CODE_SUCCESS == code) { + code = pushHierarchicalPlan(pParentsGroup, pCurrentGroup); + ++(*pLevel); + } + + if (TSDB_CODE_SUCCESS == code) { + SNode* pChild; + FOREACH(pChild, pSubplan->pChildren) { + code = doScaleOut(pCxt, (SLogicSubplan*)pChild, pLevel, pCurrentGroup); + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + } + + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyList(pCurrentGroup); + } + + return code; +} + +static SQueryLogicPlan* makeQueryLogicPlan() { + SQueryLogicPlan* pLogicPlan = (SQueryLogicPlan*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN); + if (NULL == pLogicPlan) { + return NULL; + } + pLogicPlan->pTopSubplans = nodesMakeList(); + if (NULL == pLogicPlan->pTopSubplans) { + nodesDestroyNode(pLogicPlan); + return NULL; + } + return pLogicPlan; +} + +int32_t scaleOutLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SQueryLogicPlan** pLogicPlan) { + SQueryLogicPlan* pPlan = makeQueryLogicPlan(); + if (NULL == pPlan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SScaleOutContext cxt = { .pPlanCxt = pCxt, .subplanId = 1 }; + int32_t code = doScaleOut(&cxt, pLogicSubplan, &(pPlan->totalLevel), pPlan->pTopSubplans); + if (TSDB_CODE_SUCCESS == code) { + *pLogicPlan = pPlan; + } else { + nodesDestroyNode(pPlan); + } + + return code; +} diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 4d7d31879bec6c3d37bb9ee6f49386ec87ced48c..b7a99d365d76c5e5f2b1ca949e557b2c145c249c 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -29,7 +29,7 @@ typedef struct SSplitContext { void* pInfo; } SSplitContext; -typedef int32_t (*FMatch)(SSplitContext* pCxt, SSubLogicPlan* pSubplan); +typedef int32_t (*FMatch)(SSplitContext* pCxt, SLogicSubplan* pSubplan); typedef int32_t (*FSplit)(SSplitContext* pCxt); typedef struct SSplitRule { @@ -40,7 +40,7 @@ typedef struct SSplitRule { typedef struct SStsInfo { SScanLogicNode* pScan; - SSubLogicPlan* pSubplan; + SLogicSubplan* pSubplan; } SStsInfo; static SLogicNode* stsMatchByNode(SLogicNode* pNode) { @@ -58,13 +58,13 @@ static SLogicNode* stsMatchByNode(SLogicNode* pNode) { return NULL; } -static int32_t stsMatch(SSplitContext* pCxt, SSubLogicPlan* pSubplan) { +static int32_t stsMatch(SSplitContext* pCxt, SLogicSubplan* pSubplan) { if (SPLIT_FLAG_TEST_MASK(pSubplan->splitFlag, SPLIT_FLAG_STS)) { return TSDB_CODE_SUCCESS; } SLogicNode* pSplitNode = stsMatchByNode(pSubplan->pNode); if (NULL != pSplitNode) { - SStsInfo* pInfo = calloc(1, sizeof(SStsInfo)); + SStsInfo* pInfo = taosMemoryCalloc(1, sizeof(SStsInfo)); CHECK_ALLOC(pInfo, TSDB_CODE_OUT_OF_MEMORY); pInfo->pScan = (SScanLogicNode*)pSplitNode; pInfo->pSubplan = pSubplan; @@ -74,7 +74,7 @@ static int32_t stsMatch(SSplitContext* pCxt, SSubLogicPlan* pSubplan) { } SNode* pChild; FOREACH(pChild, pSubplan->pChildren) { - int32_t code = stsMatch(pCxt, (SSubLogicPlan*)pChild); + int32_t code = stsMatch(pCxt, (SLogicSubplan*)pChild); if (TSDB_CODE_SUCCESS != code || pCxt->match) { return code; } @@ -82,8 +82,8 @@ static int32_t stsMatch(SSplitContext* pCxt, SSubLogicPlan* pSubplan) { return TSDB_CODE_SUCCESS; } -static SSubLogicPlan* stsCreateScanSubplan(SSplitContext* pCxt, SScanLogicNode* pScan) { - SSubLogicPlan* pSubplan = nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); +static SLogicSubplan* stsCreateScanSubplan(SSplitContext* pCxt, SScanLogicNode* pScan) { + SLogicSubplan* pSubplan = nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); if (NULL == pSubplan) { return NULL; } @@ -95,7 +95,7 @@ static SSubLogicPlan* stsCreateScanSubplan(SSplitContext* pCxt, SScanLogicNode* return pSubplan; } -static int32_t stsCreateExchangeNode(SSplitContext* pCxt, SSubLogicPlan* pSubplan, SScanLogicNode* pScan) { +static int32_t stsCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SScanLogicNode* pScan) { SExchangeLogicNode* pExchange = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_EXCHANGE); if (NULL == pExchange) { return TSDB_CODE_OUT_OF_MEMORY; @@ -106,6 +106,8 @@ static int32_t stsCreateExchangeNode(SSplitContext* pCxt, SSubLogicPlan* pSubpla return TSDB_CODE_OUT_OF_MEMORY; } + pSubplan->subplanType = SUBPLAN_TYPE_MERGE; + if (NULL == pScan->node.pParent) { pSubplan->pNode = (SLogicNode*)pExchange; return TSDB_CODE_SUCCESS; @@ -145,7 +147,7 @@ static const SSplitRule splitRuleSet[] = { static const int32_t splitRuleNum = (sizeof(splitRuleSet) / sizeof(SSplitRule)); -int32_t applySplitRule(SSubLogicPlan* pSubplan) { +static int32_t applySplitRule(SLogicSubplan* pSubplan) { SSplitContext cxt = { .errCode = TSDB_CODE_SUCCESS, .groupId = pSubplan->id.groupId + 1, .match = false, .pInfo = NULL }; bool split = false; do { @@ -164,3 +166,45 @@ int32_t applySplitRule(SSubLogicPlan* pSubplan) { } while (split); return TSDB_CODE_SUCCESS; } + +static void doSetLogicNodeParent(SLogicNode* pNode, SLogicNode* pParent) { + pNode->pParent = pParent; + SNode* pChild; + FOREACH(pChild, pNode->pChildren) { + doSetLogicNodeParent((SLogicNode*)pChild, pNode); + } +} + +static void setLogicNodeParent(SLogicNode* pNode) { + doSetLogicNodeParent(pNode, NULL); +} + +int32_t splitLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SLogicSubplan** pLogicSubplan) { + SLogicSubplan* pSubplan = (SLogicSubplan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); + if (NULL == pSubplan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pSubplan->pNode = nodesCloneNode(pLogicNode); + if (NULL == pSubplan->pNode) { + nodesDestroyNode(pSubplan); + return TSDB_CODE_OUT_OF_MEMORY; + } + if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIF == nodeType(pLogicNode)) { + pSubplan->subplanType = SUBPLAN_TYPE_MODIFY; + TSWAP(((SVnodeModifLogicNode*)pLogicNode)->pDataBlocks, ((SVnodeModifLogicNode*)pSubplan->pNode)->pDataBlocks, SArray*); + } else { + pSubplan->subplanType = SUBPLAN_TYPE_SCAN; + } + pSubplan->id.queryId = pCxt->queryId; + setLogicNodeParent(pSubplan->pNode); + + int32_t code = applySplitRule(pSubplan); + if (TSDB_CODE_SUCCESS == code) { + *pLogicSubplan = pSubplan; + } else { + nodesDestroyNode(pSubplan); + } + + return code; +} \ No newline at end of file diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index fa0dc549c8a0d3ea1dcaa4120a9c9e0d020cb3b5..bcea94278ea7c686e44045ea9470152e5052556c 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -17,20 +17,29 @@ #include "planInt.h" -int32_t optimize(SPlanContext* pCxt, SLogicNode* pLogicNode) { - return TSDB_CODE_SUCCESS; -} - int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNodeList) { SLogicNode* pLogicNode = NULL; + SLogicSubplan* pLogicSubplan = NULL; + SQueryLogicPlan* pLogicPlan = NULL; + int32_t code = createLogicPlan(pCxt, &pLogicNode); if (TSDB_CODE_SUCCESS == code) { - code = optimize(pCxt, pLogicNode); + code = optimizeLogicPlan(pCxt, pLogicNode); + } + if (TSDB_CODE_SUCCESS == code) { + code = splitLogicPlan(pCxt, pLogicNode, &pLogicSubplan); + } + if (TSDB_CODE_SUCCESS == code) { + code = scaleOutLogicPlan(pCxt, pLogicSubplan, &pLogicPlan); } if (TSDB_CODE_SUCCESS == code) { - code = createPhysiPlan(pCxt, pLogicNode, pPlan, pExecNodeList); + code = createPhysiPlan(pCxt, pLogicPlan, pPlan, pExecNodeList); } + nodesDestroyNode(pLogicNode); + nodesDestroyNode(pLogicSubplan); + nodesDestroyNode(pLogicPlan); + terrno = code; return code; } diff --git a/source/libs/planner/test/plannerTest.cpp b/source/libs/planner/test/plannerTest.cpp index 90d8f05efc153c0332082d507a88087e8f6d5443..b4c0e43a2d97693a9b42087f985fbb17873f52b5 100644 --- a/source/libs/planner/test/plannerTest.cpp +++ b/source/libs/planner/test/plannerTest.cpp @@ -26,11 +26,6 @@ using namespace testing; class PlannerTest : public Test { protected: - enum TestTarget { - TEST_LOGIC_PLAN, - TEST_PHYSICAL_PLAN - }; - void setDatabase(const string& acctId, const string& db) { acctId_ = acctId; db_ = db; @@ -46,7 +41,7 @@ protected: cxt_.pSql = sqlBuf_.c_str(); } - bool run(TestTarget target = TEST_PHYSICAL_PLAN) { + bool run(bool streamQuery = false) { int32_t code = qParseQuerySql(&cxt_, &query_); if (code != TSDB_CODE_SUCCESS) { @@ -56,37 +51,54 @@ protected: const string syntaxTreeStr = toString(query_->pRoot, false); - SLogicNode* pLogicPlan = nullptr; - SPlanContext cxt = { .queryId = 1, .acctId = 0 }; + SLogicNode* pLogicNode = nullptr; + SPlanContext cxt = {0}; + cxt.queryId = 1; + cxt.acctId = 0; + cxt.streamQuery = streamQuery; + setPlanContext(query_, &cxt); - code = createLogicPlan(&cxt, &pLogicPlan); + code = createLogicPlan(&cxt, &pLogicNode); if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] logic plan code:" << code << ", strerror:" << tstrerror(code) << endl; + cout << "sql:[" << cxt_.pSql << "] createLogicPlan code:" << code << ", strerror:" << tstrerror(code) << endl; return false; } cout << "====================sql : [" << cxt_.pSql << "]" << endl; - cout << "syntax test : " << endl; + cout << "syntax tree : " << endl; cout << syntaxTreeStr << endl; cout << "unformatted logic plan : " << endl; - cout << toString((const SNode*)pLogicPlan, false) << endl; - - if (TEST_PHYSICAL_PLAN == target) { - SQueryPlan* pPlan = nullptr; - code = createPhysiPlan(&cxt, pLogicPlan, &pPlan, NULL); - if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] physical plan code:" << code << ", strerror:" << tstrerror(code) << endl; - return false; - } - cout << "unformatted physical plan : " << endl; - cout << toString((const SNode*)pPlan, false) << endl; - SNode* pNode; - FOREACH(pNode, pPlan->pSubplans) { - SNode* pSubplan; - FOREACH(pSubplan, ((SNodeListNode*)pNode)->pNodeList) { - cout << "unformatted physical subplan : " << endl; - cout << toString(pSubplan, false) << endl; - } + cout << toString((const SNode*)pLogicNode, false) << endl; + + SLogicSubplan* pLogicSubplan = nullptr; + code = splitLogicPlan(&cxt, pLogicNode, &pLogicSubplan); + if (code != TSDB_CODE_SUCCESS) { + cout << "sql:[" << cxt_.pSql << "] splitLogicPlan code:" << code << ", strerror:" << tstrerror(code) << endl; + return false; + } + + SQueryLogicPlan* pLogicPlan = NULL; + code = scaleOutLogicPlan(&cxt, pLogicSubplan, &pLogicPlan); + if (code != TSDB_CODE_SUCCESS) { + cout << "sql:[" << cxt_.pSql << "] createPhysiPlan code:" << code << ", strerror:" << tstrerror(code) << endl; + return false; + } + + SQueryPlan* pPlan = nullptr; + code = createPhysiPlan(&cxt, pLogicPlan, &pPlan, NULL); + if (code != TSDB_CODE_SUCCESS) { + cout << "sql:[" << cxt_.pSql << "] createPhysiPlan code:" << code << ", strerror:" << tstrerror(code) << endl; + return false; + } + + cout << "unformatted physical plan : " << endl; + cout << toString((const SNode*)pPlan, false) << endl; + SNode* pNode; + FOREACH(pNode, pPlan->pSubplans) { + SNode* pSubplan; + FOREACH(pSubplan, ((SNodeListNode*)pNode)->pNodeList) { + cout << "unformatted physical subplan : " << endl; + cout << toString(pSubplan, false) << endl; } } @@ -100,6 +112,11 @@ private: if (QUERY_NODE_CREATE_TOPIC_STMT == nodeType(pQuery->pRoot)) { pCxt->pAstRoot = ((SCreateTopicStmt*)pQuery->pRoot)->pQuery; pCxt->topicQuery = true; + } else if (QUERY_NODE_CREATE_INDEX_STMT == nodeType(pQuery->pRoot)) { + SMCreateSmaReq req = {0}; + tDeserializeSMCreateSmaReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req); + nodesStringToNode(req.ast, &pCxt->pAstRoot); + pCxt->streamQuery = true; } else { pCxt->pAstRoot = pQuery->pRoot; } @@ -120,16 +137,8 @@ private: cout << "sql:[" << cxt_.pSql << "] toString code:" << code << ", strerror:" << tstrerror(code) << endl; return string(); } - SNode* pNode; - code = nodesStringToNode(pStr, &pNode); - if (code != TSDB_CODE_SUCCESS) { - tfree(pStr); - cout << "sql:[" << cxt_.pSql << "] toObject code:" << code << ", strerror:" << tstrerror(code) << endl; - return string(); - } - nodesDestroyNode(pNode); string str(pStr); - tfree(pStr); + taosMemoryFreeClear(pStr); return str; } @@ -185,6 +194,13 @@ TEST_F(PlannerTest, interval) { ASSERT_TRUE(run()); } +TEST_F(PlannerTest, sessionWindow) { + setDatabase("root", "test"); + + bind("SELECT count(*) FROM t1 session(ts, 10s)"); + ASSERT_TRUE(run()); +} + TEST_F(PlannerTest, showTables) { setDatabase("root", "test"); @@ -197,3 +213,17 @@ TEST_F(PlannerTest, createTopic) { bind("create topic tp as SELECT * FROM st1"); ASSERT_TRUE(run()); } + +TEST_F(PlannerTest, stream) { + setDatabase("root", "test"); + + bind("SELECT sum(c1) FROM st1"); + ASSERT_TRUE(run(true)); +} + +TEST_F(PlannerTest, createSmaIndex) { + setDatabase("root", "test"); + + bind("create sma index index1 on t1 function(max(c1), min(c3 + 10), sum(c4)) INTERVAL(10s)"); + ASSERT_TRUE(run()); +} diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index b4763024dc9ffa8a47ffbbecb22486be573c0fda..20eb49ed330cc3d742c27086085a4de728b688b2 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -28,7 +28,7 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { memcpy(pOut->db, usedbRsp->db, TSDB_DB_FNAME_LEN); pOut->dbId = usedbRsp->uid; - pOut->dbVgroup = calloc(1, sizeof(SDBVgInfo)); + pOut->dbVgroup = taosMemoryCalloc(1, sizeof(SDBVgInfo)); if (NULL == pOut->dbVgroup) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -151,7 +151,7 @@ PROCESS_USEDB_OVER: if (code != 0) { if (pOut) { if (pOut->dbVgroup) taosHashCleanup(pOut->dbVgroup->vgHash); - tfree(pOut->dbVgroup); + taosMemoryFreeClear(pOut->dbVgroup); } qError("failed to process usedb rsp since %s", terrstr()); } @@ -199,7 +199,7 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isSuperTable, STabl int32_t total = msg->numOfColumns + msg->numOfTags; int32_t metaSize = sizeof(STableMeta) + sizeof(SSchema) * total; - STableMeta *pTableMeta = calloc(1, metaSize); + STableMeta *pTableMeta = taosMemoryCalloc(1, metaSize); if (NULL == pTableMeta) { qError("calloc size[%d] failed", metaSize); return TSDB_CODE_TSC_OUT_OF_MEMORY; diff --git a/source/libs/qcom/test/queryTest.cpp b/source/libs/qcom/test/queryTest.cpp index 72ce0f7c37f814e0457699ad2c1e7b1d883bab6b..9615557c883f6adf4a22f6a1d14417e4c5fe9839 100644 --- a/source/libs/qcom/test/queryTest.cpp +++ b/source/libs/qcom/test/queryTest.cpp @@ -32,13 +32,13 @@ typedef struct SParam { int32_t testPrint(void* p) { SParam* param = (SParam*)p; printf("hello world, %d\n", param->v); - tfree(p); + taosMemoryFreeClear(p); return 0; } int32_t testPrintError(void* p) { SParam* param = (SParam*) p; - tfree(p); + taosMemoryFreeClear(p); return -1; } @@ -61,14 +61,14 @@ int main(int argc, char** argv) { } TEST(testCase, async_task_test) { - SParam* p = (SParam*)calloc(1, sizeof(SParam)); + SParam* p = (SParam*)taosMemoryCalloc(1, sizeof(SParam)); taosAsyncExec(testPrint, p, NULL); taosMsleep(5); } TEST(testCase, many_async_task_test) { for(int32_t i = 0; i < 50; ++i) { - SParam* p = (SParam*) calloc(1, sizeof(SParam)); + SParam* p = (SParam*) taosMemoryCalloc(1, sizeof(SParam)); p->v = i; taosAsyncExec(testPrint, p, NULL); } @@ -78,7 +78,7 @@ TEST(testCase, many_async_task_test) { TEST(testCase, error_in_async_test) { int32_t code = 0; - SParam* p = (SParam*) calloc(1, sizeof(SParam)); + SParam* p = (SParam*) taosMemoryCalloc(1, sizeof(SParam)); taosAsyncExec(testPrintError, p, &code); taosMsleep(1); printf("Error code:%d after asynchronously exec function\n", code); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index ae2dccfe397f1a5c2dc4e4ac3d2240ed1de45402..641fe82a853f570422778cbd06f9cd50e69f8b86 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -1259,7 +1259,7 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { return; } - rspList = calloc(schNum, sizeof(SQWHbInfo)); + rspList = taosMemoryCalloc(schNum, sizeof(SQWHbInfo)); if (NULL == rspList) { QW_UNLOCK(QW_READ, &mgmt->schLock); QW_ELOG("calloc %d SQWHbInfo failed", schNum); @@ -1293,7 +1293,7 @@ _return: tFreeSSchedulerHbRsp(&rspList[j].rsp); } - tfree(rspList); + taosMemoryFreeClear(rspList); taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); } @@ -1305,7 +1305,7 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW } int32_t code = 0; - SQWorkerMgmt *mgmt = calloc(1, sizeof(SQWorkerMgmt)); + SQWorkerMgmt *mgmt = taosMemoryCalloc(1, sizeof(SQWorkerMgmt)); if (NULL == mgmt) { qError("calloc %d failed", (int32_t)sizeof(SQWorkerMgmt)); QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1330,7 +1330,7 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW mgmt->schHash = taosHashInit(mgmt->cfg.maxSchedulerNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); if (NULL == mgmt->schHash) { - tfree(mgmt); + taosMemoryFreeClear(mgmt); qError("init %d scheduler hash failed", mgmt->cfg.maxSchedulerNum); QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -1370,7 +1370,7 @@ _return: taosTmrCleanUp(mgmt->timer); - tfree(mgmt); + taosMemoryFreeClear(mgmt); QW_RET(code); } @@ -1389,7 +1389,7 @@ void qWorkerDestroy(void **qWorkerMgmt) { //TODO FREE ALL - tfree(*qWorkerMgmt); + taosMemoryFreeClear(*qWorkerMgmt); } int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRsp **rsp) { @@ -1406,7 +1406,7 @@ int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRs taskNum = taosHashGetSize(sch->tasksHash); int32_t size = sizeof(SSchedulerStatusRsp) + sizeof((*rsp)->status[0]) * taskNum; - *rsp = calloc(1, size); + *rsp = taosMemoryCalloc(1, size); if (NULL == *rsp) { QW_SCH_ELOG("calloc %d failed", size); QW_UNLOCK(QW_READ, &sch->tasksLock); diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index 79832b4db468e216cd13de1a07dd49d08d40739a..56267c462b4ce8f0fb85aaa0a7badadf0f9511d4 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -167,7 +167,7 @@ int32_t qwBuildAndSendShowRsp(SRpcMsg *pMsg, int32_t code) { SVShowTablesRsp showRsp = {0}; // showRsp.showId = 1; - showRsp.tableMeta.pSchemas = calloc(numOfCols, sizeof(SSchema)); + showRsp.tableMeta.pSchemas = taosMemoryCalloc(numOfCols, sizeof(SSchema)); if (showRsp.tableMeta.pSchemas == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -298,7 +298,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { char* sql = strndup(msg->msg, msg->sqlLen); QW_SCH_TASK_DLOG("processQuery start, node:%p, sql:%s", node, sql); - tfree(sql); + taosMemoryFreeClear(sql); QW_ERR_RET(qwProcessQuery(QW_FPARAMS(), &qwMsg, msg->taskType)); diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 36f1e8d9001eb9c6e8628693a20c3b890fd52408..705f0e2fbe8cd9a5d063a22549425e583d8d49f0 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -165,7 +165,7 @@ int32_t qwtStringToPlan(const char* str, SSubplan** subplan) { int32_t qwtPutReqToFetchQueue(void *node, struct SRpcMsg *pMsg) { taosWLockLatch(&qwtTestFetchQueueLock); - struct SRpcMsg *newMsg = (struct SRpcMsg *)calloc(1, sizeof(struct SRpcMsg)); + struct SRpcMsg *newMsg = (struct SRpcMsg *)taosMemoryCalloc(1, sizeof(struct SRpcMsg)); memcpy(newMsg, pMsg, sizeof(struct SRpcMsg)); qwtTestFetchQueue[qwtTestFetchQueueWIdx++] = newMsg; if (qwtTestFetchQueueWIdx >= qwtTestFetchQueueSize) { @@ -188,7 +188,7 @@ int32_t qwtPutReqToFetchQueue(void *node, struct SRpcMsg *pMsg) { int32_t qwtPutReqToQueue(void *node, struct SRpcMsg *pMsg) { taosWLockLatch(&qwtTestQueryQueueLock); - struct SRpcMsg *newMsg = (struct SRpcMsg *)calloc(1, sizeof(struct SRpcMsg)); + struct SRpcMsg *newMsg = (struct SRpcMsg *)taosMemoryCalloc(1, sizeof(struct SRpcMsg)); memcpy(newMsg, pMsg, sizeof(struct SRpcMsg)); qwtTestQueryQueue[qwtTestQueryQueueWIdx++] = newMsg; if (qwtTestQueryQueueWIdx >= qwtTestQueryQueueSize) { @@ -312,7 +312,7 @@ int32_t qwtExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { } if (endExec) { - *pRes = (SSDataBlock*)calloc(1, sizeof(SSDataBlock)); + *pRes = (SSDataBlock*)taosMemoryCalloc(1, sizeof(SSDataBlock)); (*pRes)->info.rows = taosRand() % 1000 + 1; } else { *pRes = NULL; @@ -336,7 +336,7 @@ int32_t qwtPutDataBlock(DataSinkHandle handle, const SInputData* pInput, bool* p assert(0); } - free((void *)pInput->pData); + taosMemoryFree((void *)pInput->pData); taosWLockLatch(&qwtTestSinkLock); @@ -763,7 +763,7 @@ void *queryQueueThread(void *param) { assert(0); } - free(queryRpc); + taosMemoryFree(queryRpc); if (qwtTestStop && qwtTestQueryQueueNum <= 0 && qwtTestCaseFinished) { break; @@ -832,7 +832,7 @@ void *fetchQueueThread(void *param) { break; } - free(fetchRpc); + taosMemoryFree(fetchRpc); if (qwtTestStop && qwtTestFetchQueueNum <= 0 && qwtTestCaseFinished) { break; diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index e1ffb2efd12cdd00f8cdd59de5c60576239c34c8..834a722bd80e6f8374adf7a3c2ac685c843a61fb 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -338,7 +338,7 @@ typedef struct SFilterInfo { #define FILTER_PUSH_VAR_HASH(colInfo, ha) do { (colInfo).type = RANGE_TYPE_VAR_HASH; (colInfo).info = ha;} while (0) #define FILTER_PUSH_CTX(colInfo, ctx) do { (colInfo).type = RANGE_TYPE_MR_CTX; (colInfo).info = ctx;} while (0) -#define FILTER_COPY_IDX(dst, src, n) do { *(dst) = malloc(sizeof(uint32_t) * n); memcpy(*(dst), src, sizeof(uint32_t) * n);} while (0) +#define FILTER_COPY_IDX(dst, src, n) do { *(dst) = taosMemoryMalloc(sizeof(uint32_t) * n); memcpy(*(dst), src, sizeof(uint32_t) * n);} while (0) #define FILTER_ADD_CTX_TO_GRES(gres, idx, ctx) do { if ((gres)->colCtxs == NULL) { (gres)->colCtxs = taosArrayInit(gres->colNum, sizeof(SFilterColCtx)); } SFilterColCtx cCtx = {idx, ctx}; taosArrayPush((gres)->colCtxs, &cCtx); } while (0) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 58b5c8340a2152d3335d411be2d2cd58f0fd0caf..23e3e352c86622ab64cf1a27004247355edf7ac9 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -297,14 +297,14 @@ static FORCE_INLINE int32_t filterCompareGroupCtx(const void *pLeft, const void int32_t filterInitUnitsFields(SFilterInfo *info) { info->unitSize = FILTER_DEFAULT_UNIT_SIZE; - info->units = calloc(info->unitSize, sizeof(SFilterUnit)); + info->units = taosMemoryCalloc(info->unitSize, sizeof(SFilterUnit)); info->fields[FLD_TYPE_COLUMN].num = 0; info->fields[FLD_TYPE_COLUMN].size = FILTER_DEFAULT_FIELD_SIZE; - info->fields[FLD_TYPE_COLUMN].fields = calloc(info->fields[FLD_TYPE_COLUMN].size, sizeof(SFilterField)); + info->fields[FLD_TYPE_COLUMN].fields = taosMemoryCalloc(info->fields[FLD_TYPE_COLUMN].size, sizeof(SFilterField)); info->fields[FLD_TYPE_VALUE].num = 0; info->fields[FLD_TYPE_VALUE].size = FILTER_DEFAULT_FIELD_SIZE; - info->fields[FLD_TYPE_VALUE].fields = calloc(info->fields[FLD_TYPE_VALUE].size, sizeof(SFilterField)); + info->fields[FLD_TYPE_VALUE].fields = taosMemoryCalloc(info->fields[FLD_TYPE_VALUE].size, sizeof(SFilterField)); return TSDB_CODE_SUCCESS; } @@ -318,7 +318,7 @@ static FORCE_INLINE SFilterRangeNode* filterNewRange(SFilterRangeCtx *ctx, SFilt r->prev = NULL; r->next = NULL; } else { - r = calloc(1, sizeof(SFilterRangeNode)); + r = taosMemoryCalloc(1, sizeof(SFilterRangeNode)); } FILTER_COPY_RA(&r->ra, ra); @@ -332,7 +332,7 @@ void* filterInitRangeCtx(int32_t type, int32_t options) { return NULL; } - SFilterRangeCtx *ctx = calloc(1, sizeof(SFilterRangeCtx)); + SFilterRangeCtx *ctx = taosMemoryCalloc(1, sizeof(SFilterRangeCtx)); ctx->type = type; ctx->options = options; @@ -748,18 +748,18 @@ int32_t filterFreeRangeCtx(void* h) { while (r) { rn = r->next; - free(r); + taosMemoryFree(r); r = rn; } r = ctx->rf; while (r) { rn = r->next; - free(r); + taosMemoryFree(r); r = rn; } - free(ctx); + taosMemoryFree(ctx); return TSDB_CODE_SUCCESS; } @@ -769,7 +769,7 @@ int32_t filterDetachCnfGroup(SFilterGroup *gp1, SFilterGroup *gp2, SArray* group SFilterGroup gp = {0}; gp.unitNum = gp1->unitNum + gp2->unitNum; - gp.unitIdxs = calloc(gp.unitNum, sizeof(*gp.unitIdxs)); + gp.unitIdxs = taosMemoryCalloc(gp.unitNum, sizeof(*gp.unitIdxs)); memcpy(gp.unitIdxs, gp1->unitIdxs, gp1->unitNum * sizeof(*gp.unitIdxs)); memcpy(gp.unitIdxs + gp1->unitNum, gp2->unitIdxs, gp2->unitNum * sizeof(*gp.unitIdxs)); @@ -849,7 +849,7 @@ int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t d return -1; } -// In the params, we should use void *data instead of void **data, there is no need to use tfree(*data) to set *data = 0 +// In the params, we should use void *data instead of void **data, there is no need to use taosMemoryFreeClear(*data) to set *data = 0 // Besides, fields data value is a pointer, so dataLen should be POINTER_BYTES for better. int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, SFilterFieldId *fid, int32_t dataLen, bool freeIfExists) { int32_t idx = -1; @@ -869,7 +869,7 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, idx = *num; if (idx >= info->fields[type].size) { info->fields[type].size += FILTER_DEFAULT_FIELD_SIZE; - info->fields[type].fields = realloc(info->fields[type].fields, info->fields[type].size * sizeof(SFilterField)); + info->fields[type].fields = taosMemoryRealloc(info->fields[type].fields, info->fields[type].size * sizeof(SFilterField)); } info->fields[type].fields[idx].flag = type; @@ -891,7 +891,7 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, } } else { if (data && freeIfExists) { - tfree(*data); + taosMemoryFreeClear(*data); } } @@ -954,7 +954,7 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi if (info->unitNum >= info->unitSize) { uint32_t psize = info->unitSize; info->unitSize += FILTER_DEFAULT_UNIT_SIZE; - info->units = realloc(info->units, info->unitSize * sizeof(SFilterUnit)); + info->units = taosMemoryRealloc(info->units, info->unitSize * sizeof(SFilterUnit)); memset(info->units + psize, 0, sizeof(*info->units) * FILTER_DEFAULT_UNIT_SIZE); } @@ -1000,7 +1000,7 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi int32_t filterAddUnitToGroup(SFilterGroup *group, uint32_t unitIdx) { if (group->unitNum >= group->unitSize) { group->unitSize += FILTER_DEFAULT_UNIT_SIZE; - group->unitIdxs = realloc(group->unitIdxs, group->unitSize * sizeof(*group->unitIdxs)); + group->unitIdxs = taosMemoryRealloc(group->unitIdxs, group->unitSize * sizeof(*group->unitIdxs)); } group->unitIdxs[group->unitNum++] = unitIdx; @@ -1028,12 +1028,12 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) { in.type = valueNode->node.resType.type; in.bytes = valueNode->node.resType.bytes; in.data = nodesGetValueFromNode(valueNode); - out.data = malloc(sizeof(int64_t)); + out.data = taosMemoryMalloc(sizeof(int64_t)); code = vectorConvertImpl(&in, &out); if (code) { fltError("convert from %d to %d failed", in.type, out.type); - tfree(out.data); + taosMemoryFreeClear(out.data); FLT_ERR_RET(code); } @@ -1148,17 +1148,17 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan if ((!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) && (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL))) { __compar_fn_t func = getComparFunc(type, 0); if (func(&ra->s, &ra->e) == 0) { - void *data = malloc(sizeof(int64_t)); + void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); filterAddUnit(dst, OP_TYPE_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); return TSDB_CODE_SUCCESS; } else { - void *data = malloc(sizeof(int64_t)); + void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - void *data2 = malloc(sizeof(int64_t)); + void *data2 = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data2, &ra->e); filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); @@ -1170,7 +1170,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan } if (!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) { - void *data = malloc(sizeof(int64_t)); + void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, &right, &uidx); @@ -1178,7 +1178,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan } if (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { - void *data = malloc(sizeof(int64_t)); + void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->e); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); filterAddUnit(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_LOWER_THAN : OP_TYPE_LOWER_EQUAL, &left, &right, &uidx); @@ -1224,16 +1224,16 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan if ((!FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) &&(!FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL))) { __compar_fn_t func = getComparFunc(type, 0); if (func(&r->ra.s, &r->ra.e) == 0) { - void *data = malloc(sizeof(int64_t)); + void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); filterAddUnit(dst, OP_TYPE_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); } else { - void *data = malloc(sizeof(int64_t)); + void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - void *data2 = malloc(sizeof(int64_t)); + void *data2 = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data2, &r->ra.e); filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); @@ -1250,7 +1250,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan } if (!FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) { - void *data = malloc(sizeof(int64_t)); + void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, &right, &uidx); @@ -1258,7 +1258,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan } if (!FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL)) { - void *data = malloc(sizeof(int64_t)); + void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.e); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); filterAddUnit(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_LOWER_THAN : OP_TYPE_LOWER_EQUAL, &left, &right, &uidx); @@ -1284,8 +1284,8 @@ static void filterFreeGroup(void *pItem) { } SFilterGroup* p = (SFilterGroup*) pItem; - tfree(p->unitIdxs); - tfree(p->unitFlags); + taosMemoryFreeClear(p->unitIdxs); + taosMemoryFreeClear(p->unitFlags); } @@ -1635,7 +1635,7 @@ void filterFreeGroupCtx(SFilterGroupCtx* gRes) { return; } - tfree(gRes->colIdx); + taosMemoryFreeClear(gRes->colIdx); int16_t i = 0, j = 0; @@ -1648,8 +1648,8 @@ void filterFreeGroupCtx(SFilterGroupCtx* gRes) { ++j; } - tfree(gRes->colInfo); - tfree(gRes); + taosMemoryFreeClear(gRes->colInfo); + taosMemoryFreeClear(gRes); } void filterFreeField(SFilterField* field, int32_t type) { @@ -1661,7 +1661,7 @@ void filterFreeField(SFilterField* field, int32_t type) { if (FILTER_GET_FLAG(field->flag, FLD_DATA_IS_HASH)) { taosHashCleanup(field->data); } else { - tfree(field->data); + taosMemoryFreeClear(field->data); } } } @@ -1676,40 +1676,40 @@ void filterFreeInfo(SFilterInfo *info) { return; } - tfree(info->cunits); - tfree(info->blkUnitRes); - tfree(info->blkUnits); + taosMemoryFreeClear(info->cunits); + taosMemoryFreeClear(info->blkUnitRes); + taosMemoryFreeClear(info->blkUnits); for (int32_t i = 0; i < FLD_TYPE_MAX; ++i) { for (uint32_t f = 0; f < info->fields[i].num; ++f) { filterFreeField(&info->fields[i].fields[f], i); } - tfree(info->fields[i].fields); + taosMemoryFreeClear(info->fields[i].fields); } for (uint32_t i = 0; i < info->groupNum; ++i) { filterFreeGroup(&info->groups[i]); } - tfree(info->groups); + taosMemoryFreeClear(info->groups); - tfree(info->units); + taosMemoryFreeClear(info->units); - tfree(info->unitRes); + taosMemoryFreeClear(info->unitRes); - tfree(info->unitFlags); + taosMemoryFreeClear(info->unitFlags); for (uint32_t i = 0; i < info->colRangeNum; ++i) { filterFreeRangeCtx(info->colRange[i]); } - tfree(info->colRange); + taosMemoryFreeClear(info->colRange); filterFreePCtx(&info->pctx); if (!FILTER_GET_FLAG(info->status, FI_STATUS_CLONED)) { - tfree(info); + taosMemoryFreeClear(info); } } @@ -1775,14 +1775,14 @@ int32_t fltInitValFieldData(SFilterInfo *info) { if (type == TSDB_DATA_TYPE_BINARY) { size_t len = (dType->type == TSDB_DATA_TYPE_BINARY || dType->type == TSDB_DATA_TYPE_NCHAR) ? dType->bytes : MAX_NUM_STR_SIZE; - fi->data = calloc(1, len + 1 + VARSTR_HEADER_SIZE); + fi->data = taosMemoryCalloc(1, len + 1 + VARSTR_HEADER_SIZE); } else if (type == TSDB_DATA_TYPE_NCHAR) { size_t len = (dType->type == TSDB_DATA_TYPE_BINARY || dType->type == TSDB_DATA_TYPE_NCHAR) ? dType->bytes : MAX_NUM_STR_SIZE; - fi->data = calloc(1, (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); + fi->data = taosMemoryCalloc(1, (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); } else if (type != TSDB_DATA_TYPE_JSON){ if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE /* - fi->data = calloc(dType->bytes, tDataTypes[type].bytes); + fi->data = taosMemoryCalloc(dType->bytes, tDataTypes[type].bytes); for (int32_t a = 0; a < dType->bytes; ++a) { int64_t *v = taosArrayGet(var->arr, a); assignVal((char *)fi->data + a * tDataTypes[type].bytes, (char *)v, 0, type); @@ -1790,7 +1790,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { */ continue; } else { - fi->data = calloc(1, sizeof(int64_t)); + fi->data = taosMemoryCalloc(1, sizeof(int64_t)); } } else{ // type == TSDB_DATA_TYPE_JSON // fi->data = null; use fi->desc as data, because json value is variable, so use tVariant (fi->desc) @@ -1998,15 +1998,15 @@ _return: int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t* gResNum) { bool empty = false; - uint32_t *colIdx = malloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint32_t)); + uint32_t *colIdx = taosMemoryMalloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint32_t)); uint32_t colIdxi = 0; uint32_t gResIdx = 0; for (uint32_t i = 0; i < info->groupNum; ++i) { SFilterGroup* g = info->groups + i; - gRes[gResIdx] = calloc(1, sizeof(SFilterGroupCtx)); - gRes[gResIdx]->colInfo = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(SFilterColInfo)); + gRes[gResIdx] = taosMemoryCalloc(1, sizeof(SFilterGroupCtx)); + gRes[gResIdx]->colInfo = taosMemoryCalloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(SFilterColInfo)); colIdxi = 0; empty = false; @@ -2058,7 +2058,7 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t ++gResIdx; } - tfree(colIdx); + taosMemoryFreeClear(colIdx); *gResNum = gResIdx; @@ -2350,12 +2350,12 @@ int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray* group) { info->groupNum = (uint32_t)groupSize; if (info->groupNum > 0) { - info->groups = calloc(info->groupNum, sizeof(*info->groups)); + info->groups = taosMemoryCalloc(info->groupNum, sizeof(*info->groups)); } for (size_t i = 0; i < groupSize; ++i) { SFilterGroup *pg = taosArrayGet(group, i); - pg->unitFlags = calloc(pg->unitNum, sizeof(*pg->unitFlags)); + pg->unitFlags = taosMemoryCalloc(pg->unitNum, sizeof(*pg->unitFlags)); info->groups[i] = *pg; } @@ -2435,7 +2435,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ uint32_t *idxs = NULL; uint32_t colNum = 0; SFilterGroupCtx *res = NULL; - uint32_t *idxNum = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxNum)); + uint32_t *idxNum = taosMemoryCalloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxNum)); for (int32_t i = 0; i < gResNum; ++i) { for (uint32_t m = 0; m < gRes[i]->colNum; ++m) { @@ -2456,7 +2456,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ assert(idxNum[i] == gResNum); if (idxs == NULL) { - idxs = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxs)); + idxs = taosMemoryCalloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxs)); } idxs[colNum++] = i; @@ -2465,7 +2465,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ FLT_CHK_JMP(colNum <= 0); info->colRangeNum = colNum; - info->colRange = calloc(colNum, POINTER_BYTES); + info->colRange = taosMemoryCalloc(colNum, POINTER_BYTES); for (int32_t i = 0; i < gResNum; ++i) { res = gRes[i]; @@ -2512,8 +2512,8 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ } _return: - tfree(idxNum); - tfree(idxs); + taosMemoryFreeClear(idxNum); + taosMemoryFreeClear(idxs); return TSDB_CODE_SUCCESS; } @@ -2533,9 +2533,9 @@ int32_t filterPostProcessRange(SFilterInfo *info) { int32_t filterGenerateComInfo(SFilterInfo *info) { - info->cunits = malloc(info->unitNum * sizeof(*info->cunits)); - info->blkUnitRes = malloc(sizeof(*info->blkUnitRes) * info->unitNum); - info->blkUnits = malloc(sizeof(*info->blkUnits) * (info->unitNum + 1) * info->groupNum); + info->cunits = taosMemoryMalloc(info->unitNum * sizeof(*info->cunits)); + info->blkUnitRes = taosMemoryMalloc(sizeof(*info->blkUnitRes) * info->unitNum); + info->blkUnits = taosMemoryMalloc(sizeof(*info->blkUnits) * (info->unitNum + 1) * info->groupNum); for (uint32_t i = 0; i < info->unitNum; ++i) { SFilterUnit *unit = &info->units[i]; @@ -2766,7 +2766,7 @@ bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, uint32_t *unitIdx = NULL; if (*p == NULL) { - *p = calloc(numOfRows, sizeof(int8_t)); + *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); } for (int32_t i = 0; i < numOfRows; ++i) { @@ -2871,7 +2871,7 @@ static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows, } if (*p == NULL) { - *p = calloc(numOfRows, sizeof(int8_t)); + *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); } for (int32_t i = 0; i < numOfRows; ++i) { @@ -2905,7 +2905,7 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows } if (*p == NULL) { - *p = calloc(numOfRows, sizeof(int8_t)); + *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); } for (int32_t i = 0; i < numOfRows; ++i) { @@ -2947,7 +2947,7 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD } if (*p == NULL) { - *p = calloc(numOfRows, sizeof(int8_t)); + *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); } for (int32_t i = 0; i < numOfRows; ++i) { @@ -2977,7 +2977,7 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa } if (*p == NULL) { - *p = calloc(numOfRows, sizeof(int8_t)); + *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); } for (int32_t i = 0; i < numOfRows; ++i) { @@ -2991,7 +2991,7 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa // match/nmatch for nchar type need convert from ucs4 to mbs if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_NCHAR && (info->cunits[uidx].optr == OP_TYPE_MATCH || info->cunits[uidx].optr == OP_TYPE_NMATCH)){ - char *newColData = calloc(info->cunits[uidx].dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); + char *newColData = taosMemoryCalloc(info->cunits[uidx].dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(colData), varDataLen(colData), varDataVal(newColData)); if (len < 0){ qError("castConvert1 taosUcs4ToMbs error"); @@ -2999,7 +2999,7 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa varDataSetLen(newColData, len); (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, newColData, info->cunits[uidx].valData); } - tfree(newColData); + taosMemoryFreeClear(newColData); }else{ (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData); } @@ -3022,7 +3022,7 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg } if (*p == NULL) { - *p = calloc(numOfRows, sizeof(int8_t)); + *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); } for (int32_t i = 0; i < numOfRows; ++i) { @@ -3051,7 +3051,7 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg (*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); } else { if(cunit->dataType == TSDB_DATA_TYPE_NCHAR && (cunit->optr == OP_TYPE_MATCH || cunit->optr == OP_TYPE_NMATCH)){ - char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); + char *newColData = taosMemoryCalloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(colData), varDataLen(colData), varDataVal(newColData)); if (len < 0){ qError("castConvert1 taosUcs4ToMbs error"); @@ -3059,7 +3059,7 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg varDataSetLen(newColData, len); (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, cunit->valData); } - tfree(newColData); + taosMemoryFreeClear(newColData); }else{ (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); } @@ -3125,7 +3125,7 @@ int32_t filterSetExecFunc(SFilterInfo *info) { int32_t filterPreprocess(SFilterInfo *info) { - SFilterGroupCtx** gRes = calloc(info->groupNum, sizeof(SFilterGroupCtx *)); + SFilterGroupCtx** gRes = taosMemoryCalloc(info->groupNum, sizeof(SFilterGroupCtx *)); int32_t gResNum = 0; filterMergeGroupUnits(info, gRes, &gResNum); @@ -3161,7 +3161,7 @@ _return: filterFreeGroupCtx(gRes[i]); } - tfree(gRes); + taosMemoryFreeClear(gRes); return TSDB_CODE_SUCCESS; } @@ -3216,8 +3216,8 @@ int32_t fltInitFromNode(SNode* tree, SFilterInfo *info, uint32_t options) { } } - info->unitRes = malloc(info->unitNum * sizeof(*info->unitRes)); - info->unitFlags = malloc(info->unitNum * sizeof(*info->unitFlags)); + info->unitRes = taosMemoryMalloc(info->unitNum * sizeof(*info->unitRes)); + info->unitFlags = taosMemoryMalloc(info->unitNum * sizeof(*info->unitFlags)); filterDumpInfoToString(info, "Final", 0); @@ -3418,7 +3418,7 @@ int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar SFilterField nfi = {0}; nfi.desc = fi->desc; int32_t bytes = FILTER_GET_COL_FIELD_SIZE(fi); - nfi.data = malloc(rows * bytes); + nfi.data = taosMemoryMalloc(rows * bytes); int32_t bufSize = bytes - VARSTR_HEADER_SIZE; for (int32_t j = 0; j < rows; ++j) { char *src = FILTER_GET_COL_FIELD_DATA(fi, j); @@ -3459,7 +3459,7 @@ int32_t filterFreeNcharColumns(SFilterInfo* info) { SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); if (type == TSDB_DATA_TYPE_NCHAR) { - tfree(fi->data); + taosMemoryFreeClear(fi->data); } } @@ -3634,7 +3634,7 @@ int32_t filterInitFromNode(SNode* pNode, SFilterInfo **pInfo, uint32_t options) } if (*pInfo == NULL) { - *pInfo = calloc(1, sizeof(SFilterInfo)); + *pInfo = taosMemoryCalloc(1, sizeof(SFilterInfo)); if (NULL == *pInfo) { fltError("calloc %d failed", (int32_t)sizeof(SFilterInfo)); FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 8e7eaa0f8c65818504de1eb507f86376aa7dd077..83902c9df856b53cb096e858a480bd09c25ae006 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -31,7 +31,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { SScalarParam in = {.num = 1}, out = {.num = 1, .type = type}; int8_t dummy = 0; int32_t bufLen = 60; - out.data = malloc(bufLen); + out.data = taosMemoryMalloc(bufLen); int32_t len = 0; void *buf = NULL; @@ -75,14 +75,14 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { cell = cell->pNext; } - tfree(out.data); + taosMemoryFreeClear(out.data); *data = pObj; return TSDB_CODE_SUCCESS; _return: - tfree(out.data); + taosMemoryFreeClear(out.data); taosHashCleanup(pObj); SCL_RET(code); @@ -98,7 +98,7 @@ FORCE_INLINE bool sclIsNull(SScalarParam* param, int32_t idx) { FORCE_INLINE void sclSetNull(SScalarParam* param, int32_t idx) { if (NULL == param->bitmap) { - param->bitmap = calloc(BitmapLen(param->num), sizeof(char)); + param->bitmap = taosMemoryCalloc(BitmapLen(param->num), sizeof(char)); if (NULL == param->bitmap) { sclError("calloc %d failed", param->num); return; @@ -126,7 +126,7 @@ void sclFreeRes(SHashObj *res) { } void sclFreeParamNoData(SScalarParam *param) { - tfree(param->bitmap); + taosMemoryFreeClear(param->bitmap); } @@ -137,7 +137,7 @@ void sclFreeParam(SScalarParam *param) { if (SCL_DATA_TYPE_DUMMY_HASH == param->type) { taosHashCleanup((SHashObj *)param->orig.data); } else { - tfree(param->orig.data); + taosMemoryFreeClear(param->orig.data); } } } @@ -148,7 +148,7 @@ int32_t sclCopyValueNodeValue(SValueNode *pNode, void **res) { return TSDB_CODE_SUCCESS; } - *res = malloc(pNode->node.resType.bytes); + *res = taosMemoryMalloc(pNode->node.resType.bytes); if (NULL == (*res)) { sclError("malloc %d failed", pNode->node.resType.bytes); SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -284,7 +284,7 @@ int32_t sclMoveParamListData(SScalarParam *params, int32_t listNum, int32_t idx) int32_t sclInitParamList(SScalarParam **pParams, SNodeList* pParamList, SScalarCtx *ctx, int32_t *rowNum) { int32_t code = 0; - SScalarParam *paramList = calloc(pParamList->length, sizeof(SScalarParam)); + SScalarParam *paramList = taosMemoryCalloc(pParamList->length, sizeof(SScalarParam)); if (NULL == paramList) { sclError("calloc %d failed", (int32_t)(pParamList->length * sizeof(SScalarParam))); SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -308,7 +308,7 @@ int32_t sclInitParamList(SScalarParam **pParams, SNodeList* pParamList, SScalarC _return: - tfree(paramList); + taosMemoryFreeClear(paramList); SCL_RET(code); } @@ -320,7 +320,7 @@ int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScal SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SScalarParam *paramList = calloc(paramNum, sizeof(SScalarParam)); + SScalarParam *paramList = taosMemoryCalloc(paramNum, sizeof(SScalarParam)); if (NULL == paramList) { sclError("calloc %d failed", (int32_t)(paramNum * sizeof(SScalarParam))); SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -337,7 +337,7 @@ int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScal _return: - tfree(paramList); + taosMemoryFreeClear(paramList); SCL_RET(code); } @@ -360,7 +360,7 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu SCL_ERR_RET(sclInitParamList(¶ms, node->pParameterList, ctx, &rowNum)); output->type = node->node.resType.type; - output->data = calloc(rowNum, sizeof(tDataTypes[output->type].bytes)); + output->data = taosMemoryCalloc(rowNum, sizeof(tDataTypes[output->type].bytes)); if (NULL == output->data) { sclError("calloc %d failed", (int32_t)(rowNum * sizeof(tDataTypes[output->type].bytes))); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -384,7 +384,7 @@ _return: sclFreeParamNoData(params + i); } - tfree(params); + taosMemoryFreeClear(params); SCL_RET(code); } @@ -415,7 +415,7 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o output->type = node->node.resType.type; output->bytes = sizeof(bool); output->num = rowNum; - output->data = calloc(rowNum, sizeof(bool)); + output->data = taosMemoryCalloc(rowNum, sizeof(bool)); if (NULL == output->data) { sclError("calloc %d failed", (int32_t)(rowNum * sizeof(bool))); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -449,7 +449,7 @@ _return: sclFreeParamNoData(params + i); } - tfree(params); + taosMemoryFreeClear(params); SCL_RET(code); } @@ -463,7 +463,7 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp output->type = node->node.resType.type; output->num = rowNum; output->bytes = tDataTypes[output->type].bytes; - output->data = calloc(rowNum, tDataTypes[output->type].bytes); + output->data = taosMemoryCalloc(rowNum, tDataTypes[output->type].bytes); if (NULL == output->data) { sclError("calloc %d failed", (int32_t)rowNum * tDataTypes[output->type].bytes); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -485,7 +485,7 @@ _return: sclFreeParamNoData(params + i); } - tfree(params); + taosMemoryFreeClear(params); SCL_RET(code); } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index a56f6a092551c1e3847690333c5d9a8e9b4e1ca7..10829c20e3dc86a6719010331bb8729d88f717e7 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -560,7 +560,7 @@ static void tconcat(SScalarParam* pOutput, size_t numOfInput, const SScalarParam } } - pOutput->data = realloc(pOutput->data, rowLen * num); + pOutput->data = taosMemoryRealloc(pOutput->data, rowLen * num); assert(pOutput->data); char* rstart = pOutput->data; @@ -675,13 +675,13 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncPa SScalarFuncParam rightOutput = {0}; if (pLeft->nodeType == TEXPR_BINARYEXPR_NODE || pLeft->nodeType == TEXPR_UNARYEXPR_NODE) { - leftOutput.data = malloc(sizeof(int64_t) * numOfRows); + leftOutput.data = taosMemoryMalloc(sizeof(int64_t) * numOfRows); evaluateExprNodeTree(pLeft, numOfRows, &leftOutput, param, getSourceDataBlock); } // the right output has result from the right child syntax tree if (pRight->nodeType == TEXPR_BINARYEXPR_NODE || pRight->nodeType == TEXPR_UNARYEXPR_NODE) { - rightOutput.data = malloc(sizeof(int64_t) * numOfRows); + rightOutput.data = taosMemoryMalloc(sizeof(int64_t) * numOfRows); evaluateExprNodeTree(pRight, numOfRows, &rightOutput, param, getSourceDataBlock); } @@ -713,7 +713,7 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncPa void* outputBuf = pOutput->data; if (isStringOp(pExprs->_node.optr)) { - outputBuf = realloc(pOutput->data, (left.bytes + right.bytes) * left.num); + outputBuf = taosMemoryRealloc(pOutput->data, (left.bytes + right.bytes) * left.num); } OperatorFn(&left, &right, outputBuf, TSDB_ORDER_ASC); @@ -736,7 +736,7 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncPa // reserve enough memory buffer if (isBinaryStringOp(pExprs->_node.optr)) { - void* outputBuf = realloc(pOutput->data, left.bytes * left.num); + void* outputBuf = taosMemoryRealloc(pOutput->data, left.bytes * left.num); assert(outputBuf != NULL); pOutput->data = outputBuf; } @@ -744,8 +744,8 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncPa OperatorFn(&left, pOutput); } - tfree(leftOutput.data); - tfree(rightOutput.data); + taosMemoryFreeClear(leftOutput.data); + taosMemoryFreeClear(rightOutput.data); return 0; } @@ -770,13 +770,13 @@ void setScalarFunctionSupp(struct SScalarFunctionSupport* sas, SExprInfo *pExprI return; } - sas->colList = calloc(1, pSDataBlock->info.numOfCols*sizeof(SColumnInfo)); + sas->colList = taosMemoryCalloc(1, pSDataBlock->info.numOfCols*sizeof(SColumnInfo)); for(int32_t i = 0; i < sas->numOfCols; ++i) { SColumnInfoData* pColData = taosArrayGet(pSDataBlock->pDataBlock, i); sas->colList[i] = pColData->info; } - sas->data = calloc(sas->numOfCols, POINTER_BYTES); + sas->data = taosMemoryCalloc(sas->numOfCols, POINTER_BYTES); // set the input column data for (int32_t f = 0; f < pSDataBlock->info.numOfCols; ++f) { @@ -786,7 +786,7 @@ void setScalarFunctionSupp(struct SScalarFunctionSupport* sas, SExprInfo *pExprI } SScalarFunctionSupport* createScalarFuncSupport(int32_t num) { - SScalarFunctionSupport* pSupp = calloc(num, sizeof(SScalarFunctionSupport)); + SScalarFunctionSupport* pSupp = taosMemoryCalloc(num, sizeof(SScalarFunctionSupport)); return pSupp; } @@ -797,9 +797,9 @@ void destroyScalarFuncSupport(struct SScalarFunctionSupport* pSupport, int32_t n for(int32_t i = 0; i < num; ++i) { SScalarFunctionSupport* pSupp = &pSupport[i]; - tfree(pSupp->data); - tfree(pSupp->colList); + taosMemoryFreeClear(pSupp->data); + taosMemoryFreeClear(pSupp->colList); } - tfree(pSupport); + taosMemoryFreeClear(pSupport); } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 17ac9b19fd5574b61beb25ec3198dace2992e5fe..2300f5a1d364bba6f0692a27a05df8d3fca8705e 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -305,7 +305,7 @@ int32_t vectorConvertFromVarData(SScalarParam* pIn, SScalarParam* pOut, int32_t if (TSDB_DATA_TYPE_BINARY == inType) { if (varDataLen(pIn->data) >= bufSize) { bufSize = varDataLen(pIn->data) + 1; - tmp = realloc(tmp, bufSize); + tmp = taosMemoryRealloc(tmp, bufSize); } memcpy(tmp, varDataVal(pIn->data), varDataLen(pIn->data)); @@ -313,13 +313,13 @@ int32_t vectorConvertFromVarData(SScalarParam* pIn, SScalarParam* pOut, int32_t } else { if (varDataLen(pIn->data) * TSDB_NCHAR_SIZE >= bufSize) { bufSize = varDataLen(pIn->data) * TSDB_NCHAR_SIZE + 1; - tmp = realloc(tmp, bufSize); + tmp = taosMemoryRealloc(tmp, bufSize); } int len = taosUcs4ToMbs((TdUcs4*)varDataVal(pIn->data), varDataLen(pIn->data), tmp); if (len < 0){ sclError("castConvert taosUcs4ToMbs error 1"); - tfree(tmp); + taosMemoryFreeClear(tmp); return TSDB_CODE_QRY_APP_ERROR; } @@ -329,7 +329,7 @@ int32_t vectorConvertFromVarData(SScalarParam* pIn, SScalarParam* pOut, int32_t (*func)(tmp, pOut, outType); } - tfree(tmp); + taosMemoryFreeClear(tmp); return TSDB_CODE_SUCCESS; } @@ -480,7 +480,7 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p paramOut1->bytes = param1->bytes; paramOut1->type = type; paramOut1->num = param1->num; - paramOut1->data = malloc(paramOut1->num * tDataTypes[paramOut1->type].bytes); + paramOut1->data = taosMemoryMalloc(paramOut1->num * tDataTypes[paramOut1->type].bytes); if (NULL == paramOut1->data) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -488,7 +488,7 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p code = vectorConvertImpl(param1, paramOut1); if (code) { - tfree(paramOut1->data); + taosMemoryFreeClear(paramOut1->data); return code; } } @@ -497,17 +497,17 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p paramOut2->bytes = param2->bytes; paramOut2->type = type; paramOut2->num = param2->num; - paramOut2->data = malloc(paramOut2->num * tDataTypes[paramOut2->type].bytes); + paramOut2->data = taosMemoryMalloc(paramOut2->num * tDataTypes[paramOut2->type].bytes); if (NULL == paramOut2->data) { - tfree(paramOut1->data); + taosMemoryFreeClear(paramOut1->data); return TSDB_CODE_QRY_OUT_OF_MEMORY; } paramOut2->orig.data = paramOut2->data; code = vectorConvertImpl(param2, paramOut2); if (code) { - tfree(paramOut1->data); - tfree(paramOut2->data); + taosMemoryFreeClear(paramOut1->data); + taosMemoryFreeClear(paramOut2->data); return code; } } @@ -523,7 +523,7 @@ void vectorMath(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, i SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num, .dataInBlock = false}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num, .dataInBlock = false}; if (IS_VAR_DATA_TYPE(pLeft->type)) { - leftParam.data = calloc(leftParam.num, sizeof(double)); + leftParam.data = taosMemoryCalloc(leftParam.num, sizeof(double)); if (NULL == leftParam.data) { sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; @@ -536,7 +536,7 @@ void vectorMath(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, i pLeft = &leftParam; } if (IS_VAR_DATA_TYPE(pRight->type)) { - rightParam.data = calloc(rightParam.num, sizeof(double)); + rightParam.data = taosMemoryCalloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); sclFreeParam(&leftParam); @@ -681,7 +681,7 @@ void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, in SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num, .dataInBlock = false}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num, .dataInBlock = false}; if (IS_VAR_DATA_TYPE(pLeft->type)) { - leftParam.data = calloc(leftParam.num, sizeof(double)); + leftParam.data = taosMemoryCalloc(leftParam.num, sizeof(double)); if (NULL == leftParam.data) { sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; @@ -694,7 +694,7 @@ void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, in pLeft = &leftParam; } if (IS_VAR_DATA_TYPE(pRight->type)) { - rightParam.data = calloc(rightParam.num, sizeof(double)); + rightParam.data = taosMemoryCalloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); sclFreeParam(&leftParam); @@ -774,7 +774,7 @@ void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, in SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; if (IS_VAR_DATA_TYPE(pLeft->type)) { - leftParam.data = calloc(leftParam.num, sizeof(double)); + leftParam.data = taosMemoryCalloc(leftParam.num, sizeof(double)); if (NULL == leftParam.data) { sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; @@ -787,7 +787,7 @@ void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, in pLeft = &leftParam; } if (IS_VAR_DATA_TYPE(pRight->type)) { - rightParam.data = calloc(rightParam.num, sizeof(double)); + rightParam.data = taosMemoryCalloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); sclFreeParam(&leftParam); @@ -863,7 +863,7 @@ void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOu SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; if (IS_VAR_DATA_TYPE(pLeft->type)) { - leftParam.data = calloc(leftParam.num, sizeof(double)); + leftParam.data = taosMemoryCalloc(leftParam.num, sizeof(double)); if (NULL == leftParam.data) { sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; @@ -876,7 +876,7 @@ void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOu pLeft = &leftParam; } if (IS_VAR_DATA_TYPE(pRight->type)) { - rightParam.data = calloc(rightParam.num, sizeof(double)); + rightParam.data = taosMemoryCalloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); sclFreeParam(&leftParam); @@ -953,7 +953,7 @@ void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; if (IS_VAR_DATA_TYPE(pLeft->type)) { - leftParam.data = calloc(leftParam.num, sizeof(double)); + leftParam.data = taosMemoryCalloc(leftParam.num, sizeof(double)); if (NULL == leftParam.data) { sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; @@ -966,7 +966,7 @@ void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, pLeft = &leftParam; } if (IS_VAR_DATA_TYPE(pRight->type)) { - rightParam.data = calloc(rightParam.num, sizeof(double)); + rightParam.data = taosMemoryCalloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); sclFreeParam(&leftParam); @@ -1040,7 +1040,7 @@ void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; if (IS_VAR_DATA_TYPE(pLeft->type)) { - leftParam.data = calloc(leftParam.num, sizeof(double)); + leftParam.data = taosMemoryCalloc(leftParam.num, sizeof(double)); if (NULL == leftParam.data) { sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; @@ -1053,7 +1053,7 @@ void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO pLeft = &leftParam; } if (IS_VAR_DATA_TYPE(pRight->type)) { - rightParam.data = calloc(rightParam.num, sizeof(double)); + rightParam.data = taosMemoryCalloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); sclFreeParam(&leftParam); @@ -1206,7 +1206,7 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, SScalarParam leftParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pRight->num}; if (IS_VAR_DATA_TYPE(pLeft->type)) { - leftParam.data = calloc(leftParam.num, sizeof(int64_t)); + leftParam.data = taosMemoryCalloc(leftParam.num, sizeof(int64_t)); if (NULL == leftParam.data) { sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; @@ -1219,7 +1219,7 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, pLeft = &leftParam; } if (IS_VAR_DATA_TYPE(pRight->type)) { - rightParam.data = calloc(rightParam.num, sizeof(int64_t)); + rightParam.data = taosMemoryCalloc(rightParam.num, sizeof(int64_t)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); sclFreeParam(&leftParam); @@ -1303,7 +1303,7 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, SScalarParam leftParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pRight->num}; if (IS_VAR_DATA_TYPE(pLeft->type)) { - leftParam.data = calloc(leftParam.num, sizeof(int64_t)); + leftParam.data = taosMemoryCalloc(leftParam.num, sizeof(int64_t)); if (NULL == leftParam.data) { sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; @@ -1316,7 +1316,7 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, pLeft = &leftParam; } if (IS_VAR_DATA_TYPE(pRight->type)) { - rightParam.data = calloc(rightParam.num, sizeof(int64_t)); + rightParam.data = taosMemoryCalloc(rightParam.num, sizeof(int64_t)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); sclFreeParam(&leftParam); diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 328028a1d4675e408e77a16cad9fff36d4282d90..54f82eae2d2610013d682f2ea187e72f73b67fdc 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -71,7 +71,7 @@ void flttMakeValueNode(SNode **pNode, int32_t dataType, void *value) { vnode->node.resType.type = dataType; if (IS_VAR_DATA_TYPE(dataType)) { - vnode->datum.p = (char *)malloc(varDataTLen(value)); + vnode->datum.p = (char *)taosMemoryMalloc(varDataTLen(value)); varDataCopy(vnode->datum.p, value); vnode->node.resType.bytes = varDataLen(value); } else { @@ -102,7 +102,7 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in } if (NULL == *block) { - SSDataBlock *res = (SSDataBlock *)calloc(1, sizeof(SSDataBlock)); + SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); res->info.numOfCols = 3; res->info.rows = rowNum; res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); @@ -113,7 +113,7 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in idata.info.colId = i + 1; int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)calloc(1, size); + idata.pData = (char *)taosMemoryCalloc(1, size); taosArrayPush(res->pDataBlock, &idata); } @@ -122,7 +122,7 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in idata.info.bytes = dataBytes; idata.info.colId = 3; int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)calloc(1, size); + idata.pData = (char *)taosMemoryCalloc(1, size); taosArrayPush(res->pDataBlock, &idata); blockDataEnsureCapacity(res, rowNum); @@ -150,7 +150,7 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in idata.info.bytes = dataBytes; idata.info.colId = 1 + idx; int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)calloc(1, size); + idata.pData = (char *)taosMemoryCalloc(1, size); taosArrayPush(res->pDataBlock, &idata); res->info.numOfCols++; SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); @@ -325,7 +325,7 @@ TEST(columnTest, smallint_column_greater_double_value) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); blockDataDestroy(src); nodesDestroyNode(opNode); @@ -380,7 +380,7 @@ TEST(columnTest, int_column_greater_smallint_value) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -426,7 +426,7 @@ TEST(columnTest, int_column_in_double_list) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -491,7 +491,7 @@ TEST(columnTest, binary_column_in_binary_list) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -540,7 +540,7 @@ TEST(columnTest, binary_column_like_binary) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -588,7 +588,7 @@ TEST(columnTest, binary_column_is_null) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -635,7 +635,7 @@ TEST(columnTest, binary_column_is_not_null) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -674,7 +674,7 @@ TEST(opTest, smallint_column_greater_int_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -713,7 +713,7 @@ TEST(opTest, smallint_value_add_int_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -758,7 +758,7 @@ TEST(opTest, bigint_column_multi_binary_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -801,7 +801,7 @@ TEST(opTest, smallint_column_and_binary_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -839,7 +839,7 @@ TEST(opTest, smallint_column_or_float_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -879,7 +879,7 @@ TEST(opTest, smallint_column_or_double_value) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -924,7 +924,7 @@ TEST(opTest, binary_column_is_true) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -996,7 +996,7 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(logicNode1); blockDataDestroy(src); @@ -1065,7 +1065,7 @@ TEST(filterModelogicTest, same_column_and_or_and) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(logicNode1); blockDataDestroy(src); @@ -1135,7 +1135,7 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(logicNode1); blockDataDestroy(src); @@ -1204,7 +1204,7 @@ TEST(filterModelogicTest, same_column_or_and_or) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(logicNode1); blockDataDestroy(src); @@ -1277,7 +1277,7 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(logicNode1); blockDataDestroy(src); diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index d36a11749132985f1895fc4637fb036e55c0e147..abff763ce9713a16dc70749e8aaf9266e75e2326 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -79,7 +79,7 @@ void scltInitLogFile() { void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *slotId, bool newBlock, int32_t rows, SColumnInfo *colInfo) { if (newBlock) { - SSDataBlock *res = (SSDataBlock *)calloc(1, sizeof(SSDataBlock)); + SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); res->info.numOfCols = 1; res->info.rows = rows; res->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); @@ -114,7 +114,7 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { vnode->node.resType.type = dataType; if (IS_VAR_DATA_TYPE(dataType)) { - vnode->datum.p = (char *)malloc(varDataTLen(value)); + vnode->datum.p = (char *)taosMemoryMalloc(varDataTLen(value)); varDataCopy(vnode->datum.p, value); vnode->node.resType.bytes = varDataTLen(value); } else { @@ -133,7 +133,7 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in rnode->dataBlockId = 0; if (NULL == *block) { - SSDataBlock *res = (SSDataBlock *)calloc(1, sizeof(SSDataBlock)); + SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); res->info.numOfCols = 3; res->info.rows = rowNum; res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); @@ -144,7 +144,7 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in idata.info.colId = i + 1; int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)calloc(1, size); + idata.pData = (char *)taosMemoryCalloc(1, size); taosArrayPush(res->pDataBlock, &idata); } @@ -153,7 +153,7 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in idata.info.bytes = dataBytes; idata.info.colId = 3; int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)calloc(1, size); + idata.pData = (char *)taosMemoryCalloc(1, size); taosArrayPush(res->pDataBlock, &idata); blockDataEnsureCapacity(res, rowNum); @@ -181,7 +181,7 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in idata.info.bytes = dataBytes; idata.info.colId = 1 + idx; int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)calloc(1, size); + idata.pData = (char *)taosMemoryCalloc(1, size); taosArrayPush(res->pDataBlock, &idata); res->info.numOfCols++; SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index fe102ed6ed77d16b6fde9d41772a1d2663dc5f7a..9e120041afa77c462e1fece5f01f249954234a49 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -70,7 +70,7 @@ void schFreeTask(SSchTask *pTask) { taosArrayDestroy(pTask->candidateAddrs); } - tfree(pTask->msg); + taosMemoryFreeClear(pTask->msg); if (pTask->children) { taosArrayDestroy(pTask->children); @@ -1019,7 +1019,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch if (pJob->resData) { SCH_TASK_ELOG("got fetch rsp while res already exists, res:%p", pJob->resData); - tfree(rsp); + taosMemoryFreeClear(rsp); SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); } @@ -1089,7 +1089,7 @@ _return: schReleaseJob(pParam->refId); } - tfree(param); + taosMemoryFreeClear(param); SCH_RET(code); } @@ -1203,13 +1203,13 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet SSchTrans *trans = (SSchTrans *)transport; - SMsgSendInfo *pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); + SMsgSendInfo *pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SSchCallbackParam *param = calloc(1, sizeof(SSchCallbackParam)); + SSchCallbackParam *param = taosMemoryCalloc(1, sizeof(SSchCallbackParam)); if (NULL == param) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchCallbackParam)); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1241,8 +1241,8 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet _return: - tfree(param); - tfree(pMsgSendInfo); + taosMemoryFreeClear(param); + taosMemoryFreeClear(pMsgSendInfo); SCH_RET(code); } @@ -1262,7 +1262,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, case TDMT_VND_CREATE_TABLE: case TDMT_VND_SUBMIT: { msgSize = pTask->msgLen; - msg = calloc(1, msgSize); + msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1276,7 +1276,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, uint32_t len = strlen(pJob->sql); msgSize = sizeof(SSubQueryMsg) + pTask->msgLen + len; - msg = calloc(1, msgSize); + msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1300,7 +1300,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, case TDMT_VND_RES_READY: { msgSize = sizeof(SResReadyReq); - msg = calloc(1, msgSize); + msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1317,7 +1317,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, } case TDMT_VND_FETCH: { msgSize = sizeof(SResFetchReq); - msg = calloc(1, msgSize); + msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1334,7 +1334,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, } case TDMT_VND_DROP_TASK: { msgSize = sizeof(STaskDropReq); - msg = calloc(1, msgSize); + msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1362,7 +1362,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SCH_JOB_ELOG("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - msg = calloc(1, msgSize); + msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_JOB_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1394,7 +1394,7 @@ _return: SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); - tfree(msg); + taosMemoryFreeClear(msg); SCH_RET(code); } @@ -1588,8 +1588,8 @@ void schFreeJobImpl(void *job) { taosArrayDestroy(pJob->levels); taosArrayDestroy(pJob->nodeList); - tfree(pJob->resData); - tfree(pJob); + taosMemoryFreeClear(pJob->resData); + taosMemoryFreeClear(pJob); qDebug("QID:0x%" PRIx64 " job freed, refId:%" PRIx64 ", pointer:%p", queryId, refId, pJob); } @@ -1603,7 +1603,7 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pD } int32_t code = 0; - SSchJob *pJob = calloc(1, sizeof(SSchJob)); + SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); if (NULL == pJob) { qError("QID:%" PRIx64 " calloc %d failed", pDag->queryId, (int32_t)sizeof(SSchJob)); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1790,7 +1790,7 @@ int32_t schedulerConvertDagToTaskList(SQueryPlan* pDag, SArray **pTasks) { SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SSubQueryMsg* pMsg = calloc(1, msgSize); + SSubQueryMsg* pMsg = taosMemoryCalloc(1, msgSize); pMsg->header.vgId = tInfo.addr.nodeId; @@ -1807,7 +1807,7 @@ int32_t schedulerConvertDagToTaskList(SQueryPlan* pDag, SArray **pTasks) { if (NULL == taosArrayPush(info, &tInfo)) { qError("taosArrayPush failed, idx:%d", i); - free(msg); + taosMemoryFree(msg); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } } @@ -1839,7 +1839,7 @@ int32_t schedulerCopyTask(STaskInfo *src, SArray **dst, int32_t copyNum) { info.addr = src->addr; for (int32_t i = 0; i < copyNum; ++i) { - info.msg = malloc(msgSize); + info.msg = taosMemoryMalloc(msgSize); if (NULL == info.msg) { qError("malloc %d failed", msgSize); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1851,7 +1851,7 @@ int32_t schedulerCopyTask(STaskInfo *src, SArray **dst, int32_t copyNum) { if (NULL == taosArrayPush(*dst, &info)) { qError("taosArrayPush failed, idx:%d", i); - free(info.msg); + taosMemoryFree(info.msg); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } } @@ -1931,7 +1931,7 @@ int32_t schedulerFetchRows(int64_t job, void **pData) { } if (NULL == *pData) { - SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)calloc(1, sizeof(SRetrieveTableRsp)); + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp)); if (rsp) { rsp->completed = 1; } @@ -1993,7 +1993,7 @@ void schedulerFreeTaskList(SArray *taskList) { int32_t taskNum = taosArrayGetSize(taskList); for (int32_t i = 0; i < taskNum; ++i) { STaskInfo *info = taosArrayGet(taskList, i); - tfree(info->msg); + taosMemoryFreeClear(info->msg); } taosArrayDestroy(taskList); diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index b5bd68ba86bf2974be4941def7496c4a629b4cca..b2fbf33b8a9ccea50aa81181a814ce5570b8ec65 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -95,8 +95,8 @@ void schtBuildQueryDag(SQueryPlan *dag) { SNodeListNode *scan = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); SNodeListNode *merge = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); - SSubplan *scanPlan = (SSubplan *)calloc(1, sizeof(SSubplan)); - SSubplan *mergePlan = (SSubplan *)calloc(1, sizeof(SSubplan)); + SSubplan *scanPlan = (SSubplan *)taosMemoryCalloc(1, sizeof(SSubplan)); + SSubplan *mergePlan = (SSubplan *)taosMemoryCalloc(1, sizeof(SSubplan)); scanPlan->id.queryId = qId; scanPlan->id.groupId = 0x0000000000000002; @@ -110,7 +110,7 @@ void schtBuildQueryDag(SQueryPlan *dag) { scanPlan->pChildren = NULL; scanPlan->level = 1; scanPlan->pParents = nodesMakeList(); - scanPlan->pNode = (SPhysiNode*)calloc(1, sizeof(SPhysiNode)); + scanPlan->pNode = (SPhysiNode*)taosMemoryCalloc(1, sizeof(SPhysiNode)); scanPlan->msgType = TDMT_VND_QUERY; mergePlan->id.queryId = qId; @@ -122,7 +122,7 @@ void schtBuildQueryDag(SQueryPlan *dag) { mergePlan->pChildren = nodesMakeList(); mergePlan->pParents = NULL; - mergePlan->pNode = (SPhysiNode*)calloc(1, sizeof(SPhysiNode)); + mergePlan->pNode = (SPhysiNode*)taosMemoryCalloc(1, sizeof(SPhysiNode)); mergePlan->msgType = TDMT_VND_QUERY; merge->pNodeList = nodesMakeList(); @@ -148,8 +148,8 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { SNodeListNode *scan = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); SNodeListNode *merge = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); - SSubplan *scanPlan = (SSubplan *)calloc(scanPlanNum, sizeof(SSubplan)); - SSubplan *mergePlan = (SSubplan *)calloc(1, sizeof(SSubplan)); + SSubplan *scanPlan = (SSubplan *)taosMemoryCalloc(scanPlanNum, sizeof(SSubplan)); + SSubplan *mergePlan = (SSubplan *)taosMemoryCalloc(1, sizeof(SSubplan)); merge->pNodeList = nodesMakeList(); scan->pNodeList = nodesMakeList(); @@ -173,7 +173,7 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { scanPlan[i].pChildren = NULL; scanPlan[i].level = 1; scanPlan[i].pParents = nodesMakeList(); - scanPlan[i].pNode = (SPhysiNode*)calloc(1, sizeof(SPhysiNode)); + scanPlan[i].pNode = (SPhysiNode*)taosMemoryCalloc(1, sizeof(SPhysiNode)); scanPlan[i].msgType = TDMT_VND_QUERY; nodesListAppend(scanPlan[i].pParents, (SNode*)mergePlan); @@ -190,7 +190,7 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { mergePlan->execNode.epSet.numOfEps = 0; mergePlan->pParents = NULL; - mergePlan->pNode = (SPhysiNode*)calloc(1, sizeof(SPhysiNode)); + mergePlan->pNode = (SPhysiNode*)taosMemoryCalloc(1, sizeof(SPhysiNode)); mergePlan->msgType = TDMT_VND_QUERY; nodesListAppend(merge->pNodeList, (SNode*)mergePlan); @@ -213,7 +213,7 @@ void schtBuildInsertDag(SQueryPlan *dag) { dag->pSubplans = nodesMakeList(); SNodeListNode *inserta = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); - SSubplan *insertPlan = (SSubplan *)calloc(2, sizeof(SSubplan)); + SSubplan *insertPlan = (SSubplan *)taosMemoryCalloc(2, sizeof(SSubplan)); insertPlan[0].id.queryId = qId; insertPlan[0].id.groupId = 0x0000000000000003; @@ -228,7 +228,7 @@ void schtBuildInsertDag(SQueryPlan *dag) { insertPlan[0].pChildren = NULL; insertPlan[0].pParents = NULL; insertPlan[0].pNode = NULL; - insertPlan[0].pDataSink = (SDataSinkNode*)calloc(1, sizeof(SDataSinkNode)); + insertPlan[0].pDataSink = (SDataSinkNode*)taosMemoryCalloc(1, sizeof(SDataSinkNode)); insertPlan[0].msgType = TDMT_VND_SUBMIT; insertPlan[1].id.queryId = qId; @@ -244,7 +244,7 @@ void schtBuildInsertDag(SQueryPlan *dag) { insertPlan[1].pChildren = NULL; insertPlan[1].pParents = NULL; insertPlan[1].pNode = NULL; - insertPlan[1].pDataSink = (SDataSinkNode*)calloc(1, sizeof(SDataSinkNode)); + insertPlan[1].pDataSink = (SDataSinkNode*)taosMemoryCalloc(1, sizeof(SDataSinkNode)); insertPlan[1].msgType = TDMT_VND_SUBMIT; inserta->pNodeList = nodesMakeList(); @@ -258,7 +258,7 @@ void schtBuildInsertDag(SQueryPlan *dag) { int32_t schtPlanToString(const SSubplan *subplan, char** str, int32_t* len) { - *str = (char *)calloc(1, 20); + *str = (char *)taosMemoryCalloc(1, 20); *len = 20; return 0; } @@ -312,9 +312,9 @@ void schtSetRpcSendRequest() { int32_t schtAsyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransporterId, SMsgSendInfo* pInfo) { if (pInfo) { - tfree(pInfo->param); - tfree(pInfo->msgInfo.pData); - free(pInfo); + taosMemoryFreeClear(pInfo->param); + taosMemoryFreeClear(pInfo->msgInfo.pData); + taosMemoryFree(pInfo); } return 0; } @@ -373,7 +373,7 @@ void *schtCreateFetchRspThread(void *param) { taosSsleep(1); int32_t code = 0; - SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)calloc(1, sizeof(SRetrieveTableRsp)); + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp)); rsp->completed = 1; rsp->numOfRows = 10; @@ -396,13 +396,13 @@ void *schtFetchRspThread(void *aa) { taosUsleep(1); - param = (SSchCallbackParam *)calloc(1, sizeof(*param)); + param = (SSchCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); param->queryId = schtQueryId; param->taskId = schtFetchTaskId; int32_t code = 0; - SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)calloc(1, sizeof(SRetrieveTableRsp)); + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp)); rsp->completed = 1; rsp->numOfRows = 10; @@ -484,7 +484,7 @@ void* schtRunJobThread(void *aa) { pIter = taosHashIterate(pJob->execTasks, pIter); } - param = (SSchCallbackParam *)calloc(1, sizeof(*param)); + param = (SSchCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); param->refId = queryJobRefId; param->queryId = pJob->queryId; @@ -504,7 +504,7 @@ void* schtRunJobThread(void *aa) { } - param = (SSchCallbackParam *)calloc(1, sizeof(*param)); + param = (SSchCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); param->refId = queryJobRefId; param->queryId = pJob->queryId; @@ -524,7 +524,7 @@ void* schtRunJobThread(void *aa) { } - param = (SSchCallbackParam *)calloc(1, sizeof(*param)); + param = (SSchCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); param->refId = queryJobRefId; param->queryId = pJob->queryId; @@ -544,7 +544,7 @@ void* schtRunJobThread(void *aa) { } - param = (SSchCallbackParam *)calloc(1, sizeof(*param)); + param = (SSchCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); param->refId = queryJobRefId; param->queryId = pJob->queryId; @@ -697,7 +697,7 @@ TEST(queryTest, normalCase) { SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)data; ASSERT_EQ(pRsp->completed, 1); ASSERT_EQ(pRsp->numOfRows, 10); - tfree(data); + taosMemoryFreeClear(data); data = NULL; code = schedulerFetchRows(job, &data); @@ -793,7 +793,7 @@ TEST(queryTest, flowCtrlCase) { SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)data; ASSERT_EQ(pRsp->completed, 1); ASSERT_EQ(pRsp->numOfRows, 10); - tfree(data); + taosMemoryFreeClear(data); data = NULL; code = schedulerFetchRows(job, &data); diff --git a/source/libs/stream/CMakeLists.txt b/source/libs/stream/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..572c70d31ba6140b4c368d3570e31436259ed862 --- /dev/null +++ b/source/libs/stream/CMakeLists.txt @@ -0,0 +1,16 @@ +aux_source_directory(src STREAM_SRC) +add_library(stream STATIC ${STREAM_SRC}) +target_include_directories( + stream + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/stream" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries( + stream + PRIVATE os util transport qcom executor +) + +if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +endif(${BUILD_TEST}) diff --git a/source/libs/stream/inc/tstreamInc.h b/source/libs/stream/inc/tstreamInc.h new file mode 100644 index 0000000000000000000000000000000000000000..c96901e56754da81894c0c332e5cc3666d235e3b --- /dev/null +++ b/source/libs/stream/inc/tstreamInc.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef _TSTREAM_H_ +#define _TSTREAM_H_ + +#ifdef __cplusplus +} +#endif + +#endif /* ifndef _TSTREAM_H_ */ diff --git a/source/libs/stream/src/tstream.c b/source/libs/stream/src/tstream.c new file mode 100644 index 0000000000000000000000000000000000000000..0911dc52aa467d48866a68d323dcc36be7cb7a84 --- /dev/null +++ b/source/libs/stream/src/tstream.c @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tstream.h" +#include "executor.h" + +int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId) { + SArray* pRes = NULL; + // source + if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK && pTask->sourceType != TASK_SOURCE__SCAN) return 0; + + // exec + if (pTask->execType == TASK_EXEC__EXEC) { + ASSERT(workId < pTask->exec.numOfRunners); + void* exec = pTask->exec.runners[workId].executor; + pRes = taosArrayInit(0, sizeof(SSDataBlock)); + if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { + qSetStreamInput(exec, input, inputType); + while (1) { + SSDataBlock* output; + uint64_t ts; + if (qExecTask(exec, &output, &ts) < 0) { + ASSERT(false); + } + if (output == NULL) { + break; + } + taosArrayPush(pRes, output); + } + } else if (inputType == STREAM_DATA_TYPE_SSDATA_BLOCK) { + const SArray* blocks = (const SArray*)input; + int32_t sz = taosArrayGetSize(blocks); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pBlock = taosArrayGet(blocks, i); + qSetStreamInput(exec, pBlock, inputType); + while (1) { + SSDataBlock* output; + uint64_t ts; + if (qExecTask(exec, &output, &ts) < 0) { + ASSERT(false); + } + if (output == NULL) { + break; + } + taosArrayPush(pRes, output); + } + } + } else { + ASSERT(0); + } + } else { + ASSERT(inputType == STREAM_DATA_TYPE_SSDATA_BLOCK); + pRes = (SArray*)input; + } + + // sink + if (pTask->sinkType == TASK_SINK__TABLE) { + // + } else if (pTask->sinkType == TASK_SINK__SMA) { + // + } else if (pTask->sinkType == TASK_SINK__FETCH) { + // + } else if (pTask->sinkType == TASK_SINK__SHOW) { + blockDebugShowData(pRes); + } else { + ASSERT(pTask->sinkType == TASK_SINK__NONE); + } + + // dispatch + if (pTask->dispatchType != TASK_DISPATCH__NONE) { + SStreamTaskExecReq req = { + .streamId = pTask->streamId, + .taskId = pTask->taskId, + .data = pRes, + }; + + int32_t tlen = sizeof(SMsgHead) + tEncodeSStreamTaskExecReq(NULL, &req); + void* buf = rpcMallocCont(tlen); + + if (buf == NULL) { + return -1; + } + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tEncodeSStreamTaskExecReq(&abuf, &req); + + SRpcMsg dispatchMsg = { + .pCont = buf, + .contLen = tlen, + .code = 0, + .msgType = pTask->dispatchMsgType, + }; + if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { + int32_t qType; + if (pTask->dispatchMsgType == TDMT_VND_TASK_PIPE_EXEC || pTask->dispatchMsgType == TDMT_SND_TASK_PIPE_EXEC) { + qType = FETCH_QUEUE; + } else if (pTask->dispatchMsgType == TDMT_VND_TASK_MERGE_EXEC || + pTask->dispatchMsgType == TDMT_SND_TASK_MERGE_EXEC) { + qType = MERGE_QUEUE; + } else if (pTask->dispatchMsgType == TDMT_VND_TASK_WRITE_EXEC) { + qType = WRITE_QUEUE; + } else { + ASSERT(0); + } + tmsgPutToQueue(pMsgCb, qType, &dispatchMsg); + } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { + ((SMsgHead*)buf)->vgId = pTask->fixedEpDispatcher.nodeId; + SEpSet* pEpSet = &pTask->fixedEpDispatcher.epSet; + tmsgSendReq(pMsgCb, pEpSet, &dispatchMsg); + } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { + // TODO + } else { + ASSERT(0); + } + } + return 0; +} + +int32_t tEncodeSStreamTaskExecReq(void** buf, const SStreamTaskExecReq* pReq) { + int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, pReq->streamId); + tlen += taosEncodeFixedI32(buf, pReq->taskId); + tlen += tEncodeDataBlocks(buf, pReq->data); + return tlen; +} + +void* tDecodeSStreamTaskExecReq(const void* buf, SStreamTaskExecReq* pReq) { + buf = taosDecodeFixedI64(buf, &pReq->streamId); + buf = taosDecodeFixedI32(buf, &pReq->taskId); + buf = tDecodeDataBlocks(buf, &pReq->data); + return (void*)buf; +} + +void tFreeSStreamTaskExecReq(SStreamTaskExecReq* pReq) { taosArrayDestroy(pReq->data); } + +SStreamTask* tNewSStreamTask(int64_t streamId) { + SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask)); + if (pTask == NULL) { + return NULL; + } + pTask->taskId = tGenIdPI32(); + pTask->streamId = streamId; + pTask->status = STREAM_TASK_STATUS__RUNNING; + /*pTask->qmsg = NULL;*/ + return pTask; +} + +int32_t tEncodeSStreamTask(SCoder* pEncoder, const SStreamTask* pTask) { + /*if (tStartEncode(pEncoder) < 0) return -1;*/ + if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->status) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->sourceType) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->execType) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->sinkType) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->dispatchType) < 0) return -1; + if (tEncodeI16(pEncoder, pTask->dispatchMsgType) < 0) return -1; + if (tEncodeI32(pEncoder, pTask->downstreamTaskId) < 0) return -1; + + if (pTask->execType == TASK_EXEC__EXEC) { + if (tEncodeI8(pEncoder, pTask->exec.parallelizable) < 0) return -1; + if (tEncodeCStr(pEncoder, pTask->exec.qmsg) < 0) return -1; + } + + if (pTask->sinkType != TASK_SINK__NONE) { + // TODO: wrap + if (tEncodeI8(pEncoder, pTask->tbSink.reserved) < 0) return -1; + } + + if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { + if (tEncodeI8(pEncoder, pTask->inplaceDispatcher.reserved) < 0) return -1; + } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { + if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.nodeId) < 0) return -1; + if (tEncodeSEpSet(pEncoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1; + } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { + if (tEncodeI8(pEncoder, pTask->shuffleDispatcher.hashMethod) < 0) return -1; + } + + /*tEndEncode(pEncoder);*/ + return pEncoder->pos; +} + +int32_t tDecodeSStreamTask(SCoder* pDecoder, SStreamTask* pTask) { + /*if (tStartDecode(pDecoder) < 0) return -1;*/ + if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->sourceType) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->execType) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->sinkType) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->dispatchType) < 0) return -1; + if (tDecodeI16(pDecoder, &pTask->dispatchMsgType) < 0) return -1; + if (tDecodeI32(pDecoder, &pTask->downstreamTaskId) < 0) return -1; + + if (pTask->execType == TASK_EXEC__EXEC) { + if (tDecodeI8(pDecoder, &pTask->exec.parallelizable) < 0) return -1; + if (tDecodeCStrAlloc(pDecoder, &pTask->exec.qmsg) < 0) return -1; + } + + if (pTask->sinkType != TASK_SINK__NONE) { + if (tDecodeI8(pDecoder, &pTask->tbSink.reserved) < 0) return -1; + } + + if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { + if (tDecodeI8(pDecoder, &pTask->inplaceDispatcher.reserved) < 0) return -1; + } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { + if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.nodeId) < 0) return -1; + if (tDecodeSEpSet(pDecoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1; + } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { + if (tDecodeI8(pDecoder, &pTask->shuffleDispatcher.hashMethod) < 0) return -1; + } + + /*tEndDecode(pDecoder);*/ + return 0; +} + +void tFreeSStreamTask(SStreamTask* pTask) { + // TODO + /*taosMemoryFree(pTask->qmsg);*/ + /*taosMemoryFree(pTask->executor);*/ + /*taosMemoryFree(pTask);*/ +} + +#if 0 +int32_t tEncodeSStreamTaskExecReq(SCoder* pEncoder, const SStreamTaskExecReq* pReq) { + if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1; + /*if (tEncodeDataBlocks(buf, pReq->streamId) < 0) return -1;*/ + return pEncoder->size; +} +int32_t tDecodeSStreamTaskExecReq(SCoder* pDecoder, SStreamTaskExecReq* pReq) { + return pEncoder->size; +} +void tFreeSStreamTaskExecReq(SStreamTaskExecReq* pReq) { + taosArrayDestroyEx(pReq->data, tDeleteSSDataBlock); +} +#endif diff --git a/source/libs/stream/test/CMakeLists.txt b/source/libs/stream/test/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index 037a49f45c5e7a43f77d2dfff13edf7be56398fd..d4e63e9676d519c0ae92a0d93a7199f780c58eee 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -27,6 +27,12 @@ extern "C" { #include "syncMessage.h" #include "taosdef.h" +typedef enum EntryType { + SYNC_RAFT_ENTRY_NOOP = 0, + SYNC_RAFT_ENTRY_DATA = 1, + SYNC_RAFT_ENTRY_CONFIG = 2, +} EntryType; + typedef struct SSyncRaftEntry { uint32_t bytes; uint32_t msgType; @@ -35,12 +41,15 @@ typedef struct SSyncRaftEntry { bool isWeak; SyncTerm term; SyncIndex index; + EntryType entryType; uint32_t dataLen; char data[]; } SSyncRaftEntry; SSyncRaftEntry* syncEntryBuild(uint32_t dataLen); SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index); // step 4 +SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index, EntryType entryType); +SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index); void syncEntryDestory(SSyncRaftEntry* pEntry); char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len); // step 5 SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len); // step 6 diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index d979e0df150f48ec403012d13eafb5431c5b42e6..2196d6c207430b166d6596344b9c5837f2b03d28 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -47,6 +47,8 @@ SyncIndex logStoreGetCommitIndex(SSyncLogStore* pLogStore); SSyncRaftEntry* logStoreGetLastEntry(SSyncLogStore* pLogStore); cJSON* logStore2Json(SSyncLogStore* pLogStore); char* logStore2Str(SSyncLogStore* pLogStore); +cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore); +char* logStoreSimple2Str(SSyncLogStore* pLogStore); // for debug void logStorePrint(SSyncLogStore* pLogStore); @@ -54,6 +56,11 @@ void logStorePrint2(char* s, SSyncLogStore* pLogStore); void logStoreLog(SSyncLogStore* pLogStore); void logStoreLog2(char* s, SSyncLogStore* pLogStore); +void logStoreSimplePrint(SSyncLogStore* pLogStore); +void logStoreSimplePrint2(char* s, SSyncLogStore* pLogStore); +void logStoreSimpleLog(SSyncLogStore* pLogStore); +void logStoreSimpleLog2(char* s, SSyncLogStore* pLogStore); + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index e4df93ca47c013348289f057a8a10063e6e6edfe..2e9cd63de6aa0f80daefa3f385c9b4463e4f5adc 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -120,6 +120,11 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { // reject request if ((pMsg->term < ths->pRaftStore->currentTerm) || ((pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && !logOK)) { + sTrace( + "syncNodeOnAppendEntriesCb --> reject, pMsg->term:%lu, ths->pRaftStore->currentTerm:%lu, ths->state:%d, " + "logOK:%d", + pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK); + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(); pReply->srcId = ths->myRaftId; pReply->destId = pMsg->srcId; @@ -137,6 +142,11 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { // return to follower state if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE) { + sTrace( + "syncNodeOnAppendEntriesCb --> return to follower, pMsg->term:%lu, ths->pRaftStore->currentTerm:%lu, " + "ths->state:%d, logOK:%d", + pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK); + syncNodeBecomeFollower(ths); // ret or reply? @@ -145,89 +155,63 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { // accept request if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_FOLLOWER && logOK) { - bool preMatch = false; - if (pMsg->prevLogIndex == SYNC_INDEX_INVALID && - ths->pLogStore->getLastIndex(ths->pLogStore) == SYNC_INDEX_INVALID) { - preMatch = true; - } - if (pMsg->prevLogIndex >= SYNC_INDEX_BEGIN && pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { - SSyncRaftEntry* pPreEntry = logStoreGetEntry(ths->pLogStore, pMsg->prevLogIndex); - assert(pPreEntry != NULL); - if (pMsg->prevLogTerm == pPreEntry->term) { - preMatch = true; - } - syncEntryDestory(pPreEntry); - } + // preIndex = -1, or has preIndex entry in local log + assert(pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)); - if (preMatch) { - // must has preIndex in local log - assert(pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)); + // has extra entries (> preIndex) in local log + bool hasExtraEntries = pMsg->prevLogIndex < ths->pLogStore->getLastIndex(ths->pLogStore); - bool hasExtraEntries = pMsg->prevLogIndex < ths->pLogStore->getLastIndex(ths->pLogStore); - bool hasAppendEntries = pMsg->dataLen > 0; + // has entries in SyncAppendEntries msg + bool hasAppendEntries = pMsg->dataLen > 0; - if (hasExtraEntries && hasAppendEntries) { - // conflict - bool conflict = false; + sTrace( + "syncNodeOnAppendEntriesCb --> accept, pMsg->term:%lu, ths->pRaftStore->currentTerm:%lu, ths->state:%d, " + "logOK:%d, hasExtraEntries:%d, hasAppendEntries:%d", + pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK, hasExtraEntries, hasAppendEntries); - SyncIndex extraIndex = pMsg->prevLogIndex + 1; - SSyncRaftEntry* pExtraEntry = logStoreGetEntry(ths->pLogStore, extraIndex); - assert(pExtraEntry != NULL); + if (hasExtraEntries && hasAppendEntries) { + // not conflict by default + bool conflict = false; - SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); - assert(pAppendEntry != NULL); + SyncIndex extraIndex = pMsg->prevLogIndex + 1; + SSyncRaftEntry* pExtraEntry = logStoreGetEntry(ths->pLogStore, extraIndex); + assert(pExtraEntry != NULL); - assert(extraIndex == pAppendEntry->index); - if (pExtraEntry->term == pAppendEntry->term) { - conflict = true; - } + SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + assert(pAppendEntry != NULL); - if (conflict) { - // roll back - SyncIndex delBegin = ths->pLogStore->getLastIndex(ths->pLogStore); - SyncIndex delEnd = extraIndex; + // log not match, conflict + assert(extraIndex == pAppendEntry->index); + if (pExtraEntry->term != pAppendEntry->term) { + conflict = true; + } - // notice! reverse roll back! - for (SyncIndex index = delEnd; index >= delBegin; --index) { - if (ths->pFsm->FpRollBackCb != NULL) { - SSyncRaftEntry* pRollBackEntry = logStoreGetEntry(ths->pLogStore, index); - assert(pRollBackEntry != NULL); + if (conflict) { + // roll back + SyncIndex delBegin = ths->pLogStore->getLastIndex(ths->pLogStore); + SyncIndex delEnd = extraIndex; - SRpcMsg rpcMsg; - syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg); - ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, pRollBackEntry->index, pRollBackEntry->isWeak, 0); - rpcFreeCont(rpcMsg.pCont); - syncEntryDestory(pRollBackEntry); - } - } + sTrace("syncNodeOnAppendEntriesCb --> conflict:%d, delBegin:%ld, delEnd:%ld", conflict, delBegin, delEnd); - // delete confict entries - ths->pLogStore->truncate(ths->pLogStore, extraIndex); + // notice! reverse roll back! + for (SyncIndex index = delEnd; index >= delBegin; --index) { + if (ths->pFsm->FpRollBackCb != NULL) { + SSyncRaftEntry* pRollBackEntry = logStoreGetEntry(ths->pLogStore, index); + assert(pRollBackEntry != NULL); - // append new entries - ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry); + // maybe is a NOOP ENTRY + // assert(pRollBackEntry->entryType == SYNC_RAFT_ENTRY_DATA); - // pre commit - SRpcMsg rpcMsg; - syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); - if (ths->pFsm != NULL) { - if (ths->pFsm->FpPreCommitCb != NULL) { - ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pAppendEntry->index, pAppendEntry->isWeak, 0); - } + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg); + ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, pRollBackEntry->index, pRollBackEntry->isWeak, 0, ths->state); + rpcFreeCont(rpcMsg.pCont); + syncEntryDestory(pRollBackEntry); } - rpcFreeCont(rpcMsg.pCont); } - // free memory - syncEntryDestory(pExtraEntry); - syncEntryDestory(pAppendEntry); - - } else if (hasExtraEntries && !hasAppendEntries) { - // do nothing - - } else if (!hasExtraEntries && hasAppendEntries) { - SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); - assert(pAppendEntry != NULL); + // delete confict entries + ths->pLogStore->truncate(ths->pLogStore, extraIndex); // append new entries ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry); @@ -236,54 +220,64 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); if (ths->pFsm != NULL) { - if (ths->pFsm->FpPreCommitCb != NULL) { - ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pAppendEntry->index, pAppendEntry->isWeak, 0); + if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->entryType == SYNC_RAFT_ENTRY_DATA) { + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pAppendEntry->index, pAppendEntry->isWeak, 2, ths->state); } } rpcFreeCont(rpcMsg.pCont); + } - // free memory - syncEntryDestory(pAppendEntry); + // free memory + syncEntryDestory(pExtraEntry); + syncEntryDestory(pAppendEntry); - } else if (!hasExtraEntries && !hasAppendEntries) { - // do nothing + } else if (hasExtraEntries && !hasAppendEntries) { + // do nothing - } else { - assert(0); - } + } else if (!hasExtraEntries && hasAppendEntries) { + SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + assert(pAppendEntry != NULL); - SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(); - pReply->srcId = ths->myRaftId; - pReply->destId = pMsg->srcId; - pReply->term = ths->pRaftStore->currentTerm; - pReply->success = true; + // append new entries + ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry); - if (hasAppendEntries) { - pReply->matchIndex = pMsg->prevLogIndex + 1; - } else { - pReply->matchIndex = pMsg->prevLogIndex; + // pre commit + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); + if (ths->pFsm != NULL) { + if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->entryType == SYNC_RAFT_ENTRY_DATA) { + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pAppendEntry->index, pAppendEntry->isWeak, 3, ths->state); + } } + rpcFreeCont(rpcMsg.pCont); - SRpcMsg rpcMsg; - syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); - syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + // free memory + syncEntryDestory(pAppendEntry); - syncAppendEntriesReplyDestroy(pReply); + } else if (!hasExtraEntries && !hasAppendEntries) { + // do nothing } else { - SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(); - pReply->srcId = ths->myRaftId; - pReply->destId = pMsg->srcId; - pReply->term = ths->pRaftStore->currentTerm; - pReply->success = false; - pReply->matchIndex = SYNC_INDEX_INVALID; + assert(0); + } - SRpcMsg rpcMsg; - syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); - syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); - syncAppendEntriesReplyDestroy(pReply); + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->success = true; + + if (hasAppendEntries) { + pReply->matchIndex = pMsg->prevLogIndex + 1; + } else { + pReply->matchIndex = pMsg->prevLogIndex; } + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncAppendEntriesReplyDestroy(pReply); + // maybe update commit index from leader if (pMsg->commitIndex > ths->commitIndex) { // has commit entry in local @@ -307,8 +301,8 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pEntry, &rpcMsg); - if (ths->pFsm->FpCommitCb != NULL) { - ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0); + if (ths->pFsm->FpCommitCb != NULL && pEntry->entryType == SYNC_RAFT_ENTRY_DATA) { + ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0, ths->state); } rpcFreeCont(rpcMsg.pCont); diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 817974fd268fa4105c45d51c5b591a79167971dc..790ac7f8e1c997c3f8721b5badfc85a9b6fa2a94 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -48,6 +48,9 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* p return ret; } + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplyCb== before pNextIndex", ths->pNextIndex); + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplyCb== before pMatchIndex", ths->pMatchIndex); + // no need this code, because if I receive reply.term, then I must have sent for that term. // if (pMsg->term > ths->pRaftStore->currentTerm) { // syncNodeUpdateTerm(ths, pMsg->term); @@ -77,5 +80,8 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* p syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), nextIndex); } + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplyCb== after pNextIndex", ths->pNextIndex); + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplyCb== after pMatchIndex", ths->pMatchIndex); + return ret; } diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 0d4df8e6cf43aecce88b71145f1f0f4c1fe49744..6023c00afa183716c06828470a1e91e0fdec087b 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -63,7 +63,14 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { if (pEntry->term == pSyncNode->pRaftStore->currentTerm) { // update commit index newCommitIndex = index; + sTrace("syncMaybeAdvanceCommitIndex maybe to update, newCommitIndex:%ld commit, pSyncNode->commitIndex:%ld", + newCommitIndex, pSyncNode->commitIndex); break; + } else { + sTrace( + "syncMaybeAdvanceCommitIndex can not commit due to term not equal, pEntry->term:%lu, " + "pSyncNode->pRaftStore->currentTerm:%lu", + pEntry->term, pSyncNode->pRaftStore->currentTerm); } } } @@ -90,8 +97,8 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pEntry, &rpcMsg); - if (pSyncNode->pFsm->FpCommitCb != NULL) { - pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0); + if (pSyncNode->pFsm->FpCommitCb != NULL && pEntry->entryType == SYNC_RAFT_ENTRY_DATA) { + pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0, pSyncNode->state); } rpcFreeCont(rpcMsg.pCont); diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index d47a525e2584652a700b2a9d4da2721fa0571dc2..726ebdae07de5e406ac488cce1660524f1310112 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -74,7 +74,7 @@ static void syncEnvTick(void *param, void *tmrId) { } static SSyncEnv *doSyncEnvStart() { - SSyncEnv *pSyncEnv = (SSyncEnv *)malloc(sizeof(SSyncEnv)); + SSyncEnv *pSyncEnv = (SSyncEnv *)taosMemoryMalloc(sizeof(SSyncEnv)); assert(pSyncEnv != NULL); memset(pSyncEnv, 0, sizeof(SSyncEnv)); diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 19121632e20f55e1402898f352533cf9565c0249..717b1cc7565645eec916ec5f677cf85e4b8bb2e3 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -29,7 +29,7 @@ static int32_t syncIODestroy(SSyncIO *io); static int32_t syncIOStartInternal(SSyncIO *io); static int32_t syncIOStopInternal(SSyncIO *io); -static void * syncIOConsumerFunc(void *param); +static void *syncIOConsumerFunc(void *param); static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); @@ -75,6 +75,7 @@ int32_t syncIOSendMsg(void *clientRpc, const SEpSet *pEpSet, SRpcMsg *pMsg) { syncRpcMsgPrint2(logBuf, pMsg); pMsg->handle = NULL; + pMsg->noResp = 1; rpcSendRequest(clientRpc, pEpSet, pMsg, NULL); return ret; } @@ -120,7 +121,7 @@ int32_t syncIOPingTimerStop() { // local function ------------ static SSyncIO *syncIOCreate(char *host, uint16_t port) { - SSyncIO *io = (SSyncIO *)malloc(sizeof(SSyncIO)); + SSyncIO *io = (SSyncIO *)taosMemoryMalloc(sizeof(SSyncIO)); memset(io, 0, sizeof(*io)); io->pMsgQ = taosOpenQueue(); @@ -234,9 +235,9 @@ static int32_t syncIOStopInternal(SSyncIO *io) { } static void *syncIOConsumerFunc(void *param) { - SSyncIO * io = param; + SSyncIO *io = param; STaosQall *qall; - SRpcMsg * pRpcMsg, rpcMsg; + SRpcMsg *pRpcMsg, rpcMsg; qall = taosAllocateQall(); while (1) { @@ -257,13 +258,6 @@ static void *syncIOConsumerFunc(void *param) { assert(pSyncMsg != NULL); io->FpOnSyncPing(io->pSyncNode, pSyncMsg); syncPingDestroy(pSyncMsg); - /* - pSyncMsg = syncPingBuild(pRpcMsg->contLen); - syncPingFromRpcMsg(pRpcMsg, pSyncMsg); - // memcpy(pSyncMsg, tmpRpcMsg.pCont, tmpRpcMsg.contLen); - io->FpOnSyncPing(io->pSyncNode, pSyncMsg); - syncPingDestroy(pSyncMsg); - */ } } else if (pRpcMsg->msgType == SYNC_PING_REPLY) { @@ -331,19 +325,21 @@ static void *syncIOConsumerFunc(void *param) { taosGetQitem(qall, (void **)&pRpcMsg); rpcFreeCont(pRpcMsg->pCont); - if (pRpcMsg->handle != NULL) { - int msgSize = 32; - memset(&rpcMsg, 0, sizeof(rpcMsg)); - rpcMsg.msgType = SYNC_RESPONSE; - rpcMsg.pCont = rpcMallocCont(msgSize); - rpcMsg.contLen = msgSize; - snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", "give a reply"); - rpcMsg.handle = pRpcMsg->handle; - rpcMsg.code = 0; - - syncRpcMsgPrint2((char *)"syncIOConsumerFunc rpcSendResponse --> ", &rpcMsg); - rpcSendResponse(&rpcMsg); - } + /* + if (pRpcMsg->handle != NULL) { + int msgSize = 32; + memset(&rpcMsg, 0, sizeof(rpcMsg)); + rpcMsg.msgType = SYNC_RESPONSE; + rpcMsg.pCont = rpcMallocCont(msgSize); + rpcMsg.contLen = msgSize; + snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", "give a reply"); + rpcMsg.handle = pRpcMsg->handle; + rpcMsg.code = 0; + + syncRpcMsgPrint2((char *)"syncIOConsumerFunc rpcSendResponse --> ", &rpcMsg); + rpcSendResponse(&rpcMsg); + } + */ taosFreeQitem(pRpcMsg); } diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c index aa97104180c37dfb2455a79cfef8696717ac5ec1..d33075054a888a1b2903fe230f6503e6c19aa580 100644 --- a/source/libs/sync/src/syncIndexMgr.c +++ b/source/libs/sync/src/syncIndexMgr.c @@ -19,7 +19,7 @@ // SMatchIndex ----------------------------- SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode) { - SSyncIndexMgr *pSyncIndexMgr = malloc(sizeof(SSyncIndexMgr)); + SSyncIndexMgr *pSyncIndexMgr = taosMemoryMalloc(sizeof(SSyncIndexMgr)); assert(pSyncIndexMgr != NULL); memset(pSyncIndexMgr, 0, sizeof(SSyncIndexMgr)); @@ -33,7 +33,7 @@ SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode) { void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr) { if (pSyncIndexMgr != NULL) { - free(pSyncIndexMgr); + taosMemoryFree(pSyncIndexMgr); } } @@ -78,12 +78,12 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) { cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pSyncIndexMgr->replicas))[i])); } int respondNum = 0; - int *arr = (int *)malloc(sizeof(int) * pSyncIndexMgr->replicaNum); + int *arr = (int *)taosMemoryMalloc(sizeof(int) * pSyncIndexMgr->replicaNum); for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { arr[i] = pSyncIndexMgr->index[i]; } cJSON *pIndex = cJSON_CreateIntArray(arr, pSyncIndexMgr->replicaNum); - free(arr); + taosMemoryFree(arr); cJSON_AddItemToObject(pRoot, "index", pIndex); snprintf(u64buf, sizeof(u64buf), "%p", pSyncIndexMgr->pSyncNode); cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); @@ -106,24 +106,24 @@ void syncIndexMgrPrint(SSyncIndexMgr *pObj) { char *serialized = syncIndexMgr2Str(pObj); printf("syncIndexMgrPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncIndexMgrPrint2(char *s, SSyncIndexMgr *pObj) { char *serialized = syncIndexMgr2Str(pObj); printf("syncIndexMgrPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncIndexMgrLog(SSyncIndexMgr *pObj) { char *serialized = syncIndexMgr2Str(pObj); sTrace("syncIndexMgrLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncIndexMgrLog2(char *s, SSyncIndexMgr *pObj) { char *serialized = syncIndexMgr2Str(pObj); sTrace("syncIndexMgrLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } \ No newline at end of file diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index b92317ef3f75621a665b2fbb7ec8618f75e30a2b..78e454309ac24edac140d52434cc3f2d192916d8 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -37,11 +37,13 @@ static int32_t tsNodeRefId = -1; // ------ local funciton --------- // enqueue message ---- -static void syncNodeEqPingTimer(void* param, void* tmrId); -static void syncNodeEqElectTimer(void* param, void* tmrId); -static void syncNodeEqHeartbeatTimer(void* param, void* tmrId); +static void syncNodeEqPingTimer(void* param, void* tmrId); +static void syncNodeEqElectTimer(void* param, void* tmrId); +static void syncNodeEqHeartbeatTimer(void* param, void* tmrId); +static int32_t syncNodeEqNoop(SSyncNode* ths); +static int32_t syncNodeAppendNoop(SSyncNode* ths); -// on message ---- +// process message ---- static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg); @@ -104,9 +106,26 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) { } int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { - int32_t ret = 0; + int32_t ret = syncPropose2(rid, pMsg, isWeak, 0); + return ret; +} + +int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { + int32_t ret = syncPropose(rid, pMsg, isWeak); + return ret; +} + +ESyncState syncGetMyRole(int64_t rid) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return TAOS_SYNC_STATE_ERROR; + } + assert(rid == pSyncNode->rid); + return pSyncNode->state; +} - // todo : get pointer from rid +int32_t syncPropose2(int64_t rid, const SRpcMsg* pMsg, bool isWeak, uint64_t seqNum) { + int32_t ret = 0; SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { return -1; @@ -114,7 +133,7 @@ int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { assert(rid == pSyncNode->rid); if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { - SyncClientRequest* pSyncMsg = syncClientRequestBuild2(pMsg, 0, isWeak); + SyncClientRequest* pSyncMsg = syncClientRequestBuild2(pMsg, seqNum, isWeak); SRpcMsg rpcMsg; syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg); pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); @@ -122,7 +141,7 @@ int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { ret = 0; } else { - sTrace("syncForwardToPeer not leader, %s", syncUtilState2String(pSyncNode->state)); + sTrace("syncPropose not leader, %s", syncUtilState2String(pSyncNode->state)); ret = -1; // todo : need define err code !! } @@ -130,22 +149,9 @@ int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { return ret; } -int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { - int32_t ret = syncPropose(rid, pMsg, isWeak); - return ret; -} - -ESyncState syncGetMyRole(int64_t rid) { - // todo : get pointer from rid - SSyncNode* pSyncNode = NULL; - return pSyncNode->state; -} - -void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole) {} - // open/close -------------- SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { - SSyncNode* pSyncNode = (SSyncNode*)malloc(sizeof(SSyncNode)); + SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode)); assert(pSyncNode != NULL); memset(pSyncNode, 0, sizeof(SSyncNode)); @@ -304,7 +310,7 @@ void syncNodeClose(SSyncNode* pSyncNode) { syncNodeStopElectTimer(pSyncNode); syncNodeStopHeartbeatTimer(pSyncNode); - free(pSyncNode); + taosMemoryFree(pSyncNode); } // ping -------------- @@ -665,6 +671,10 @@ void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); assert(voteGrantedMajority(pSyncNode->pVotesGranted)); syncNodeBecomeLeader(pSyncNode); + + // Raft 3.6.2 Committing entries from previous terms + syncNodeAppendNoop(pSyncNode); + // syncNodeEqNoop(pSyncNode); } void syncNodeFollower2Candidate(SSyncNode* pSyncNode) { @@ -713,26 +723,26 @@ void syncNodePrint(SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); printf("syncNodePrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncNodePrint2(char* s, SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); printf("syncNodePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncNodeLog(SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); sTrace("syncNodeLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncNodeLog2(char* s, SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); sTrace("syncNodeLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ------ local funciton --------- @@ -799,6 +809,47 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { } } +static int32_t syncNodeEqNoop(SSyncNode* ths) { + int32_t ret = 0; + assert(ths->state == TAOS_SYNC_STATE_LEADER); + + SyncIndex index = ths->pLogStore->getLastIndex(ths->pLogStore) + 1; + SyncTerm term = ths->pRaftStore->currentTerm; + SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index); + assert(pEntry != NULL); + + uint32_t entryLen; + char* serialized = syncEntrySerialize(pEntry, &entryLen); + SyncClientRequest* pSyncMsg = syncClientRequestBuild(entryLen); + assert(pSyncMsg->dataLen == entryLen); + memcpy(pSyncMsg->data, serialized, entryLen); + + SRpcMsg rpcMsg; + syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg); + ths->FpEqMsg(ths->queue, &rpcMsg); + + taosMemoryFree(serialized); + syncClientRequestDestroy(pSyncMsg); + + return ret; +} + +static int32_t syncNodeAppendNoop(SSyncNode* ths) { + int32_t ret = 0; + + SyncIndex index = ths->pLogStore->getLastIndex(ths->pLogStore) + 1; + SyncTerm term = ths->pRaftStore->currentTerm; + SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index); + assert(pEntry != NULL); + + if (ths->state == TAOS_SYNC_STATE_LEADER) { + ths->pLogStore->appendEntry(ths->pLogStore, pEntry); + syncNodeReplicate(ths); + } + + return ret; +} + // on message ---- static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) { int32_t ret = 0; @@ -847,8 +898,8 @@ static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg syncEntry2OriginalRpc(pEntry, &rpcMsg); if (ths->pFsm != NULL) { - if (ths->pFsm->FpPreCommitCb != NULL) { - ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0); + if (ths->pFsm->FpPreCommitCb != NULL && pEntry->entryType == SYNC_RAFT_ENTRY_DATA) { + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0, ths->state); } } rpcFreeCont(rpcMsg.pCont); @@ -862,8 +913,8 @@ static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg syncEntry2OriginalRpc(pEntry, &rpcMsg); if (ths->pFsm != NULL) { - if (ths->pFsm->FpPreCommitCb != NULL) { - ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, -2); + if (ths->pFsm->FpPreCommitCb != NULL && pEntry->entryType == SYNC_RAFT_ENTRY_DATA) { + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 1, ths->state); } } rpcFreeCont(rpcMsg.pCont); @@ -877,5 +928,5 @@ static void syncFreeNode(void* param) { SSyncNode* pNode = param; syncNodePrint2((char*)"==syncFreeNode==", pNode); - free(pNode); + taosMemoryFree(pNode); } \ No newline at end of file diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 3ac4a7a1c083485fe776a92a59c2989b111966a7..5dca165455240d9eaf57c8cd7e76b264fbb68f3e 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -70,20 +70,20 @@ cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) { char* s; s = syncUtilprintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen); cJSON_AddStringToObject(pRoot, "pCont", s); - free(s); + taosMemoryFree(s); s = syncUtilprintBin2((char*)(pRpcMsg->pCont), pRpcMsg->contLen); cJSON_AddStringToObject(pRoot, "pCont2", s); - free(s); + taosMemoryFree(s); } else { pRoot = cJSON_CreateObject(); char* s; s = syncUtilprintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen); cJSON_AddStringToObject(pRoot, "pCont", s); - free(s); + taosMemoryFree(s); s = syncUtilprintBin2((char*)(pRpcMsg->pCont), pRpcMsg->contLen); cJSON_AddStringToObject(pRoot, "pCont2", s); - free(s); + taosMemoryFree(s); } cJSON_AddNumberToObject(pRoot, "msgType", pRpcMsg->msgType); @@ -118,32 +118,32 @@ void syncRpcMsgPrint(SRpcMsg* pMsg) { char* serialized = syncRpcMsg2Str(pMsg); printf("syncRpcMsgPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncRpcMsgPrint2(char* s, SRpcMsg* pMsg) { char* serialized = syncRpcMsg2Str(pMsg); printf("syncRpcMsgPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncRpcMsgLog(SRpcMsg* pMsg) { char* serialized = syncRpcMsg2Str(pMsg); sTrace("syncRpcMsgLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncRpcMsgLog2(char* s, SRpcMsg* pMsg) { char* serialized = syncRpcMsg2Str(pMsg); sTrace("syncRpcMsgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ---- message process SyncTimeout---- SyncTimeout* syncTimeoutBuild() { uint32_t bytes = sizeof(SyncTimeout); - SyncTimeout* pMsg = malloc(bytes); + SyncTimeout* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_TIMEOUT; @@ -161,7 +161,7 @@ SyncTimeout* syncTimeoutBuild2(ESyncTimeoutType timeoutType, uint64_t logicClock void syncTimeoutDestroy(SyncTimeout* pMsg) { if (pMsg != NULL) { - free(pMsg); + taosMemoryFree(pMsg); } } @@ -176,7 +176,7 @@ void syncTimeoutDeserialize(const char* buf, uint32_t len, SyncTimeout* pMsg) { } char* syncTimeoutSerialize2(const SyncTimeout* pMsg, uint32_t* len) { - char* buf = malloc(pMsg->bytes); + char* buf = taosMemoryMalloc(pMsg->bytes); assert(buf != NULL); syncTimeoutSerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { @@ -187,7 +187,7 @@ char* syncTimeoutSerialize2(const SyncTimeout* pMsg, uint32_t* len) { SyncTimeout* syncTimeoutDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SyncTimeout* pMsg = malloc(bytes); + SyncTimeout* pMsg = taosMemoryMalloc(bytes); assert(pMsg != NULL); syncTimeoutDeserialize(buf, len, pMsg); assert(len == pMsg->bytes); @@ -243,32 +243,32 @@ void syncTimeoutPrint(const SyncTimeout* pMsg) { char* serialized = syncTimeout2Str(pMsg); printf("syncTimeoutPrint | len:%zu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncTimeoutPrint2(char* s, const SyncTimeout* pMsg) { char* serialized = syncTimeout2Str(pMsg); printf("syncTimeoutPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncTimeoutLog(const SyncTimeout* pMsg) { char* serialized = syncTimeout2Str(pMsg); sTrace("syncTimeoutLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncTimeoutLog2(char* s, const SyncTimeout* pMsg) { char* serialized = syncTimeout2Str(pMsg); sTrace("syncTimeoutLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ---- message process SyncPing---- SyncPing* syncPingBuild(uint32_t dataLen) { uint32_t bytes = sizeof(SyncPing) + dataLen; - SyncPing* pMsg = malloc(bytes); + SyncPing* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_PING; @@ -292,7 +292,7 @@ SyncPing* syncPingBuild3(const SRaftId* srcId, const SRaftId* destId) { void syncPingDestroy(SyncPing* pMsg) { if (pMsg != NULL) { - free(pMsg); + taosMemoryFree(pMsg); } } @@ -308,7 +308,7 @@ void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg) { } char* syncPingSerialize2(const SyncPing* pMsg, uint32_t* len) { - char* buf = malloc(pMsg->bytes); + char* buf = taosMemoryMalloc(pMsg->bytes); assert(buf != NULL); syncPingSerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { @@ -319,7 +319,7 @@ char* syncPingSerialize2(const SyncPing* pMsg, uint32_t* len) { SyncPing* syncPingDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SyncPing* pMsg = malloc(bytes); + SyncPing* pMsg = taosMemoryMalloc(bytes); assert(pMsg != NULL); syncPingDeserialize(buf, len, pMsg); assert(len == pMsg->bytes); @@ -385,10 +385,10 @@ cJSON* syncPing2Json(const SyncPing* pMsg) { char* s; s = syncUtilprintBin((char*)(pMsg->data), pMsg->dataLen); cJSON_AddStringToObject(pRoot, "data", s); - free(s); + taosMemoryFree(s); s = syncUtilprintBin2((char*)(pMsg->data), pMsg->dataLen); cJSON_AddStringToObject(pRoot, "data2", s); - free(s); + taosMemoryFree(s); } cJSON* pJson = cJSON_CreateObject(); @@ -408,32 +408,32 @@ void syncPingPrint(const SyncPing* pMsg) { char* serialized = syncPing2Str(pMsg); printf("syncPingPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncPingPrint2(char* s, const SyncPing* pMsg) { char* serialized = syncPing2Str(pMsg); printf("syncPingPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncPingLog(const SyncPing* pMsg) { char* serialized = syncPing2Str(pMsg); sTrace("syncPingLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncPingLog2(char* s, const SyncPing* pMsg) { char* serialized = syncPing2Str(pMsg); sTrace("syncPingLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ---- message process SyncPingReply---- SyncPingReply* syncPingReplyBuild(uint32_t dataLen) { uint32_t bytes = sizeof(SyncPingReply) + dataLen; - SyncPingReply* pMsg = malloc(bytes); + SyncPingReply* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_PING_REPLY; @@ -457,7 +457,7 @@ SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId) void syncPingReplyDestroy(SyncPingReply* pMsg) { if (pMsg != NULL) { - free(pMsg); + taosMemoryFree(pMsg); } } @@ -473,7 +473,7 @@ void syncPingReplyDeserialize(const char* buf, uint32_t len, SyncPingReply* pMsg } char* syncPingReplySerialize2(const SyncPingReply* pMsg, uint32_t* len) { - char* buf = malloc(pMsg->bytes); + char* buf = taosMemoryMalloc(pMsg->bytes); assert(buf != NULL); syncPingReplySerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { @@ -484,7 +484,7 @@ char* syncPingReplySerialize2(const SyncPingReply* pMsg, uint32_t* len) { SyncPingReply* syncPingReplyDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SyncPingReply* pMsg = malloc(bytes); + SyncPingReply* pMsg = taosMemoryMalloc(bytes); assert(pMsg != NULL); syncPingReplyDeserialize(buf, len, pMsg); assert(len == pMsg->bytes); @@ -550,10 +550,10 @@ cJSON* syncPingReply2Json(const SyncPingReply* pMsg) { char* s; s = syncUtilprintBin((char*)(pMsg->data), pMsg->dataLen); cJSON_AddStringToObject(pRoot, "data", s); - free(s); + taosMemoryFree(s); s = syncUtilprintBin2((char*)(pMsg->data), pMsg->dataLen); cJSON_AddStringToObject(pRoot, "data2", s); - free(s); + taosMemoryFree(s); } cJSON* pJson = cJSON_CreateObject(); @@ -573,32 +573,32 @@ void syncPingReplyPrint(const SyncPingReply* pMsg) { char* serialized = syncPingReply2Str(pMsg); printf("syncPingReplyPrint | len:%zu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncPingReplyPrint2(char* s, const SyncPingReply* pMsg) { char* serialized = syncPingReply2Str(pMsg); printf("syncPingReplyPrint2 | len:%zu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncPingReplyLog(const SyncPingReply* pMsg) { char* serialized = syncPingReply2Str(pMsg); sTrace("syncPingReplyLog | len:%zu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncPingReplyLog2(char* s, const SyncPingReply* pMsg) { char* serialized = syncPingReply2Str(pMsg); sTrace("syncPingReplyLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ---- message process SyncClientRequest---- SyncClientRequest* syncClientRequestBuild(uint32_t dataLen) { uint32_t bytes = sizeof(SyncClientRequest) + dataLen; - SyncClientRequest* pMsg = malloc(bytes); + SyncClientRequest* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_CLIENT_REQUEST; @@ -620,7 +620,7 @@ SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint6 void syncClientRequestDestroy(SyncClientRequest* pMsg) { if (pMsg != NULL) { - free(pMsg); + taosMemoryFree(pMsg); } } @@ -635,7 +635,7 @@ void syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientReque } char* syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len) { - char* buf = malloc(pMsg->bytes); + char* buf = taosMemoryMalloc(pMsg->bytes); assert(buf != NULL); syncClientRequestSerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { @@ -646,7 +646,7 @@ char* syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len) SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SyncClientRequest* pMsg = malloc(bytes); + SyncClientRequest* pMsg = taosMemoryMalloc(bytes); assert(pMsg != NULL); syncClientRequestDeserialize(buf, len, pMsg); assert(len == pMsg->bytes); @@ -688,10 +688,10 @@ cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg) { char* s; s = syncUtilprintBin((char*)(pMsg->data), pMsg->dataLen); cJSON_AddStringToObject(pRoot, "data", s); - free(s); + taosMemoryFree(s); s = syncUtilprintBin2((char*)(pMsg->data), pMsg->dataLen); cJSON_AddStringToObject(pRoot, "data2", s); - free(s); + taosMemoryFree(s); } cJSON* pJson = cJSON_CreateObject(); @@ -711,32 +711,32 @@ void syncClientRequestPrint(const SyncClientRequest* pMsg) { char* serialized = syncClientRequest2Str(pMsg); printf("syncClientRequestPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncClientRequestPrint2(char* s, const SyncClientRequest* pMsg) { char* serialized = syncClientRequest2Str(pMsg); printf("syncClientRequestPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncClientRequestLog(const SyncClientRequest* pMsg) { char* serialized = syncClientRequest2Str(pMsg); sTrace("syncClientRequestLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg) { char* serialized = syncClientRequest2Str(pMsg); sTrace("syncClientRequestLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ---- message process SyncRequestVote---- SyncRequestVote* syncRequestVoteBuild() { uint32_t bytes = sizeof(SyncRequestVote); - SyncRequestVote* pMsg = malloc(bytes); + SyncRequestVote* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_REQUEST_VOTE; @@ -745,7 +745,7 @@ SyncRequestVote* syncRequestVoteBuild() { void syncRequestVoteDestroy(SyncRequestVote* pMsg) { if (pMsg != NULL) { - free(pMsg); + taosMemoryFree(pMsg); } } @@ -760,7 +760,7 @@ void syncRequestVoteDeserialize(const char* buf, uint32_t len, SyncRequestVote* } char* syncRequestVoteSerialize2(const SyncRequestVote* pMsg, uint32_t* len) { - char* buf = malloc(pMsg->bytes); + char* buf = taosMemoryMalloc(pMsg->bytes); assert(buf != NULL); syncRequestVoteSerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { @@ -771,7 +771,7 @@ char* syncRequestVoteSerialize2(const SyncRequestVote* pMsg, uint32_t* len) { SyncRequestVote* syncRequestVoteDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SyncRequestVote* pMsg = malloc(bytes); + SyncRequestVote* pMsg = taosMemoryMalloc(bytes); assert(pMsg != NULL); syncRequestVoteDeserialize(buf, len, pMsg); assert(len == pMsg->bytes); @@ -834,7 +834,7 @@ cJSON* syncRequestVote2Json(const SyncRequestVote* pMsg) { snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastLogIndex); + snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->lastLogIndex); cJSON_AddStringToObject(pRoot, "lastLogIndex", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastLogTerm); cJSON_AddStringToObject(pRoot, "lastLogTerm", u64buf); @@ -857,32 +857,32 @@ void syncRequestVotePrint(const SyncRequestVote* pMsg) { char* serialized = syncRequestVote2Str(pMsg); printf("syncRequestVotePrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncRequestVotePrint2(char* s, const SyncRequestVote* pMsg) { char* serialized = syncRequestVote2Str(pMsg); printf("syncRequestVotePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncRequestVoteLog(const SyncRequestVote* pMsg) { char* serialized = syncRequestVote2Str(pMsg); sTrace("syncRequestVoteLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncRequestVoteLog2(char* s, const SyncRequestVote* pMsg) { char* serialized = syncRequestVote2Str(pMsg); sTrace("syncRequestVoteLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ---- message process SyncRequestVoteReply---- SyncRequestVoteReply* syncRequestVoteReplyBuild() { uint32_t bytes = sizeof(SyncRequestVoteReply); - SyncRequestVoteReply* pMsg = malloc(bytes); + SyncRequestVoteReply* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_REQUEST_VOTE_REPLY; @@ -891,7 +891,7 @@ SyncRequestVoteReply* syncRequestVoteReplyBuild() { void syncRequestVoteReplyDestroy(SyncRequestVoteReply* pMsg) { if (pMsg != NULL) { - free(pMsg); + taosMemoryFree(pMsg); } } @@ -906,7 +906,7 @@ void syncRequestVoteReplyDeserialize(const char* buf, uint32_t len, SyncRequestV } char* syncRequestVoteReplySerialize2(const SyncRequestVoteReply* pMsg, uint32_t* len) { - char* buf = malloc(pMsg->bytes); + char* buf = taosMemoryMalloc(pMsg->bytes); assert(buf != NULL); syncRequestVoteReplySerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { @@ -917,7 +917,7 @@ char* syncRequestVoteReplySerialize2(const SyncRequestVoteReply* pMsg, uint32_t* SyncRequestVoteReply* syncRequestVoteReplyDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SyncRequestVoteReply* pMsg = malloc(bytes); + SyncRequestVoteReply* pMsg = taosMemoryMalloc(bytes); assert(pMsg != NULL); syncRequestVoteReplyDeserialize(buf, len, pMsg); assert(len == pMsg->bytes); @@ -1000,32 +1000,32 @@ void syncRequestVoteReplyPrint(const SyncRequestVoteReply* pMsg) { char* serialized = syncRequestVoteReply2Str(pMsg); printf("syncRequestVoteReplyPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncRequestVoteReplyPrint2(char* s, const SyncRequestVoteReply* pMsg) { char* serialized = syncRequestVoteReply2Str(pMsg); printf("syncRequestVoteReplyPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncRequestVoteReplyLog(const SyncRequestVoteReply* pMsg) { char* serialized = syncRequestVoteReply2Str(pMsg); sTrace("syncRequestVoteReplyLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncRequestVoteReplyLog2(char* s, const SyncRequestVoteReply* pMsg) { char* serialized = syncRequestVoteReply2Str(pMsg); sTrace("syncRequestVoteReplyLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ---- message process SyncAppendEntries---- SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen) { uint32_t bytes = sizeof(SyncAppendEntries) + dataLen; - SyncAppendEntries* pMsg = malloc(bytes); + SyncAppendEntries* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_APPEND_ENTRIES; @@ -1035,7 +1035,7 @@ SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen) { void syncAppendEntriesDestroy(SyncAppendEntries* pMsg) { if (pMsg != NULL) { - free(pMsg); + taosMemoryFree(pMsg); } } @@ -1051,7 +1051,7 @@ void syncAppendEntriesDeserialize(const char* buf, uint32_t len, SyncAppendEntri } char* syncAppendEntriesSerialize2(const SyncAppendEntries* pMsg, uint32_t* len) { - char* buf = malloc(pMsg->bytes); + char* buf = taosMemoryMalloc(pMsg->bytes); assert(buf != NULL); syncAppendEntriesSerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { @@ -1062,7 +1062,7 @@ char* syncAppendEntriesSerialize2(const SyncAppendEntries* pMsg, uint32_t* len) SyncAppendEntries* syncAppendEntriesDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SyncAppendEntries* pMsg = malloc(bytes); + SyncAppendEntries* pMsg = taosMemoryMalloc(bytes); assert(pMsg != NULL); syncAppendEntriesDeserialize(buf, len, pMsg); assert(len == pMsg->bytes); @@ -1127,23 +1127,23 @@ cJSON* syncAppendEntries2Json(const SyncAppendEntries* pMsg) { snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->prevLogIndex); - cJSON_AddStringToObject(pRoot, "pre_log_index", u64buf); + snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->prevLogIndex); + cJSON_AddStringToObject(pRoot, "prevLogIndex", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->prevLogTerm); cJSON_AddStringToObject(pRoot, "pre_log_term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->commitIndex); - cJSON_AddStringToObject(pRoot, "commit_index", u64buf); + snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->commitIndex); + cJSON_AddStringToObject(pRoot, "commitIndex", u64buf); cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); char* s; s = syncUtilprintBin((char*)(pMsg->data), pMsg->dataLen); cJSON_AddStringToObject(pRoot, "data", s); - free(s); + taosMemoryFree(s); s = syncUtilprintBin2((char*)(pMsg->data), pMsg->dataLen); cJSON_AddStringToObject(pRoot, "data2", s); - free(s); + taosMemoryFree(s); } cJSON* pJson = cJSON_CreateObject(); @@ -1163,32 +1163,32 @@ void syncAppendEntriesPrint(const SyncAppendEntries* pMsg) { char* serialized = syncAppendEntries2Str(pMsg); printf("syncAppendEntriesPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncAppendEntriesPrint2(char* s, const SyncAppendEntries* pMsg) { char* serialized = syncAppendEntries2Str(pMsg); printf("syncAppendEntriesPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncAppendEntriesLog(const SyncAppendEntries* pMsg) { char* serialized = syncAppendEntries2Str(pMsg); sTrace("syncAppendEntriesLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg) { char* serialized = syncAppendEntries2Str(pMsg); sTrace("syncAppendEntriesLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ---- message process SyncAppendEntriesReply---- SyncAppendEntriesReply* syncAppendEntriesReplyBuild() { uint32_t bytes = sizeof(SyncAppendEntriesReply); - SyncAppendEntriesReply* pMsg = malloc(bytes); + SyncAppendEntriesReply* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_APPEND_ENTRIES_REPLY; @@ -1197,7 +1197,7 @@ SyncAppendEntriesReply* syncAppendEntriesReplyBuild() { void syncAppendEntriesReplyDestroy(SyncAppendEntriesReply* pMsg) { if (pMsg != NULL) { - free(pMsg); + taosMemoryFree(pMsg); } } @@ -1212,7 +1212,7 @@ void syncAppendEntriesReplyDeserialize(const char* buf, uint32_t len, SyncAppend } char* syncAppendEntriesReplySerialize2(const SyncAppendEntriesReply* pMsg, uint32_t* len) { - char* buf = malloc(pMsg->bytes); + char* buf = taosMemoryMalloc(pMsg->bytes); assert(buf != NULL); syncAppendEntriesReplySerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { @@ -1223,7 +1223,7 @@ char* syncAppendEntriesReplySerialize2(const SyncAppendEntriesReply* pMsg, uint3 SyncAppendEntriesReply* syncAppendEntriesReplyDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SyncAppendEntriesReply* pMsg = malloc(bytes); + SyncAppendEntriesReply* pMsg = taosMemoryMalloc(bytes); assert(pMsg != NULL); syncAppendEntriesReplyDeserialize(buf, len, pMsg); assert(len == pMsg->bytes); @@ -1288,7 +1288,7 @@ cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg) { snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); cJSON_AddStringToObject(pRoot, "term", u64buf); cJSON_AddNumberToObject(pRoot, "success", pMsg->success); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->matchIndex); + snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->matchIndex); cJSON_AddStringToObject(pRoot, "matchIndex", u64buf); } @@ -1309,24 +1309,24 @@ void syncAppendEntriesReplyPrint(const SyncAppendEntriesReply* pMsg) { char* serialized = syncAppendEntriesReply2Str(pMsg); printf("syncAppendEntriesReplyPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncAppendEntriesReplyPrint2(char* s, const SyncAppendEntriesReply* pMsg) { char* serialized = syncAppendEntriesReply2Str(pMsg); printf("syncAppendEntriesReplyPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncAppendEntriesReplyLog(const SyncAppendEntriesReply* pMsg) { char* serialized = syncAppendEntriesReply2Str(pMsg); sTrace("syncAppendEntriesReplyLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg) { char* serialized = syncAppendEntriesReply2Str(pMsg); sTrace("syncAppendEntriesReplyLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index e4c5882e9896d423709e2fcc4c8de6245ad908b4..3de39a6aa802d192c2e7ae77c5eb9a57fc9b0a33 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -18,7 +18,7 @@ SSyncRaftEntry* syncEntryBuild(uint32_t dataLen) { uint32_t bytes = sizeof(SSyncRaftEntry) + dataLen; - SSyncRaftEntry* pEntry = malloc(bytes); + SSyncRaftEntry* pEntry = taosMemoryMalloc(bytes); assert(pEntry != NULL); memset(pEntry, 0, bytes); pEntry->bytes = bytes; @@ -28,6 +28,13 @@ SSyncRaftEntry* syncEntryBuild(uint32_t dataLen) { // step 4. SyncClientRequest => SSyncRaftEntry, add term, index SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index) { + SSyncRaftEntry* pEntry = syncEntryBuild3(pMsg, term, index, SYNC_RAFT_ENTRY_DATA); + assert(pEntry != NULL); + + return pEntry; +} + +SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index, EntryType entryType) { SSyncRaftEntry* pEntry = syncEntryBuild(pMsg->dataLen); assert(pEntry != NULL); @@ -37,21 +44,32 @@ SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncInde pEntry->isWeak = pMsg->isWeak; pEntry->term = term; pEntry->index = index; + pEntry->entryType = entryType; pEntry->dataLen = pMsg->dataLen; memcpy(pEntry->data, pMsg->data, pMsg->dataLen); return pEntry; } +SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index) { + SSyncRaftEntry* pEntry = syncEntryBuild(0); + assert(pEntry != NULL); + pEntry->term = term; + pEntry->index = index; + pEntry->entryType = SYNC_RAFT_ENTRY_NOOP; + + return pEntry; +} + void syncEntryDestory(SSyncRaftEntry* pEntry) { if (pEntry != NULL) { - free(pEntry); + taosMemoryFree(pEntry); } } // step 5. SSyncRaftEntry => bin, to raft log char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len) { - char* buf = malloc(pEntry->bytes); + char* buf = taosMemoryMalloc(pEntry->bytes); assert(buf != NULL); memcpy(buf, pEntry, pEntry->bytes); if (len != NULL) { @@ -63,7 +81,7 @@ char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len) { // step 6. bin => SSyncRaftEntry, from raft log SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SSyncRaftEntry* pEntry = malloc(bytes); + SSyncRaftEntry* pEntry = taosMemoryMalloc(bytes); assert(pEntry != NULL); memcpy(pEntry, buf, len); assert(len == pEntry->bytes); @@ -85,16 +103,17 @@ cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry) { cJSON_AddStringToObject(pRoot, "term", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pEntry->index); cJSON_AddStringToObject(pRoot, "index", u64buf); + cJSON_AddNumberToObject(pRoot, "entryType", pEntry->entryType); cJSON_AddNumberToObject(pRoot, "dataLen", pEntry->dataLen); char* s; s = syncUtilprintBin((char*)(pEntry->data), pEntry->dataLen); cJSON_AddStringToObject(pRoot, "data", s); - free(s); + taosMemoryFree(s); s = syncUtilprintBin2((char*)(pEntry->data), pEntry->dataLen); cJSON_AddStringToObject(pRoot, "data2", s); - free(s); + taosMemoryFree(s); } cJSON* pJson = cJSON_CreateObject(); @@ -123,24 +142,24 @@ void syncEntryPrint(const SSyncRaftEntry* pObj) { char* serialized = syncEntry2Str(pObj); printf("syncEntryPrint | len:%zu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncEntryPrint2(char* s, const SSyncRaftEntry* pObj) { char* serialized = syncEntry2Str(pObj); printf("syncEntryPrint2 | len:%zu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncEntryLog(const SSyncRaftEntry* pObj) { char* serialized = syncEntry2Str(pObj); sTrace("syncEntryLog | len:%zu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncEntryLog2(char* s, const SSyncRaftEntry* pObj) { char* serialized = syncEntry2Str(pObj); sTrace("syncEntryLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 620e923629863a844d3b1382d43e341426e10cc7..63c6265d10672a321d4f9ae7e26e4887cde9384f 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -17,10 +17,10 @@ #include "wal.h" SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { - SSyncLogStore* pLogStore = malloc(sizeof(SSyncLogStore)); + SSyncLogStore* pLogStore = taosMemoryMalloc(sizeof(SSyncLogStore)); assert(pLogStore != NULL); - pLogStore->data = malloc(sizeof(SSyncLogStoreData)); + pLogStore->data = taosMemoryMalloc(sizeof(SSyncLogStoreData)); assert(pLogStore->data != NULL); SSyncLogStoreData* pData = pLogStore->data; @@ -34,13 +34,13 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { pLogStore->getLastTerm = logStoreLastTerm; pLogStore->updateCommitIndex = logStoreUpdateCommitIndex; pLogStore->getCommitIndex = logStoreGetCommitIndex; - return pLogStore; // to avoid compiler error + return pLogStore; } void logStoreDestory(SSyncLogStore* pLogStore) { if (pLogStore != NULL) { - free(pLogStore->data); - free(pLogStore); + taosMemoryFree(pLogStore->data); + taosMemoryFree(pLogStore); } } @@ -48,18 +48,22 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - assert(pEntry->index == logStoreLastIndex(pLogStore) + 1); + SyncIndex lastIndex = logStoreLastIndex(pLogStore); + assert(pEntry->index == lastIndex + 1); uint32_t len; char* serialized = syncEntrySerialize(pEntry, &len); assert(serialized != NULL); - int code; - code = walWrite(pWal, pEntry->index, pEntry->msgType, serialized, len); - assert(code == 0); + int code = 0; + /* + code = walWrite(pWal, pEntry->index, pEntry->entryType, serialized, len); + assert(code == 0); + */ + assert(walWrite(pWal, pEntry->index, pEntry->entryType, serialized, len) == 0); walFsync(pWal, true); - free(serialized); - return code; // to avoid compiler error + taosMemoryFree(serialized); + return code; } SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { @@ -69,7 +73,7 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) { SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); - walReadWithHandle(pWalHandle, index); + assert(walReadWithHandle(pWalHandle, index) == 0); pEntry = syncEntryDeserialize(pWalHandle->pHead->head.body, pWalHandle->pHead->head.len); assert(pEntry != NULL); @@ -83,7 +87,7 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - walRollback(pWal, fromIndex); + assert(walRollback(pWal, fromIndex) == 0); return 0; // to avoid compiler error } @@ -99,7 +103,7 @@ SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore) { SSyncRaftEntry* pLastEntry = logStoreGetLastEntry(pLogStore); if (pLastEntry != NULL) { lastTerm = pLastEntry->term; - free(pLastEntry); + taosMemoryFree(pLastEntry); } return lastTerm; } @@ -107,7 +111,7 @@ SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore) { int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - walCommit(pWal, index); + assert(walCommit(pWal, index) == 0); return 0; // to avoid compiler error } @@ -165,29 +169,84 @@ char* logStore2Str(SSyncLogStore* pLogStore) { return serialized; } +cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore) { + char u64buf[128]; + SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data; + cJSON* pRoot = cJSON_CreateObject(); + + if (pData != NULL && pData->pWal != NULL) { + snprintf(u64buf, sizeof(u64buf), "%p", pData->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); + cJSON_AddStringToObject(pRoot, "pWal", u64buf); + snprintf(u64buf, sizeof(u64buf), "%ld", logStoreLastIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", logStoreLastTerm(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncLogStoreSimple", pRoot); + return pJson; +} + +char* logStoreSimple2Str(SSyncLogStore* pLogStore) { + cJSON* pJson = logStoreSimple2Json(pLogStore); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + // for debug ----------------- void logStorePrint(SSyncLogStore* pLogStore) { char* serialized = logStore2Str(pLogStore); printf("logStorePrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void logStorePrint2(char* s, SSyncLogStore* pLogStore) { char* serialized = logStore2Str(pLogStore); printf("logStorePrint | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void logStoreLog(SSyncLogStore* pLogStore) { char* serialized = logStore2Str(pLogStore); sTrace("logStorePrint | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void logStoreLog2(char* s, SSyncLogStore* pLogStore) { char* serialized = logStore2Str(pLogStore); sTrace("logStorePrint | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } + +// for debug ----------------- +void logStoreSimplePrint(SSyncLogStore* pLogStore) { + char* serialized = logStoreSimple2Str(pLogStore); + printf("logStoreSimplePrint | len:%lu | %s \n", strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void logStoreSimplePrint2(char* s, SSyncLogStore* pLogStore) { + char* serialized = logStoreSimple2Str(pLogStore); + printf("logStoreSimplePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void logStoreSimpleLog(SSyncLogStore* pLogStore) { + char* serialized = logStoreSimple2Str(pLogStore); + sTrace("logStoreSimpleLog | len:%lu | %s", strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void logStoreSimpleLog2(char* s, SSyncLogStore* pLogStore) { + char* serialized = logStoreSimple2Str(pLogStore); + sTrace("logStoreSimpleLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); +} \ No newline at end of file diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index 3f6db129ce46de97b14da2b240b56878826f5e0e..c9054d088e1536eeb7c7f7208ab230231eb7c93c 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -26,7 +26,7 @@ static bool raftStoreFileExist(char *path); SRaftStore *raftStoreOpen(const char *path) { int32_t ret; - SRaftStore *pRaftStore = malloc(sizeof(SRaftStore)); + SRaftStore *pRaftStore = taosMemoryMalloc(sizeof(SRaftStore)); if (pRaftStore == NULL) { sError("raftStoreOpen malloc error"); return NULL; @@ -75,7 +75,7 @@ int32_t raftStoreClose(SRaftStore *pRaftStore) { assert(pRaftStore != NULL); taosCloseFile(&pRaftStore->pFile); - free(pRaftStore); + taosMemoryFree(pRaftStore); pRaftStore = NULL; return 0; } @@ -128,7 +128,7 @@ int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len) { assert(len2 < len); memset(buf, 0, len); snprintf(buf, len, "%s", serialized); - free(serialized); + taosMemoryFree(serialized); cJSON_Delete(pRoot); return 0; @@ -226,23 +226,23 @@ void raftStorePrint(SRaftStore *pObj) { char *serialized = raftStore2Str(pObj); printf("raftStorePrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void raftStorePrint2(char *s, SRaftStore *pObj) { char *serialized = raftStore2Str(pObj); printf("raftStorePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void raftStoreLog(SRaftStore *pObj) { char *serialized = raftStore2Str(pObj); sTrace("raftStoreLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void raftStoreLog2(char *s, SRaftStore *pObj) { char *serialized = raftStore2Str(pObj); sTrace("raftStoreLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 0bb701fc09a112234622a3f32830ba12e47445b7..3ced7a18bc53f274f48a2e936ab2a51a45227570 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -49,6 +49,10 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { assert(pSyncNode->state == TAOS_SYNC_STATE_LEADER); + syncIndexMgrLog2("==syncNodeAppendEntriesPeers== pNextIndex", pSyncNode->pNextIndex); + syncIndexMgrLog2("==syncNodeAppendEntriesPeers== pMatchIndex", pSyncNode->pMatchIndex); + logStoreSimpleLog2("==syncNodeAppendEntriesPeers==", pSyncNode->pLogStore); + int32_t ret = 0; for (int i = 0; i < pSyncNode->peersNum; ++i) { SRaftId* pDestId = &(pSyncNode->peersId[i]); @@ -82,7 +86,7 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { assert(len == pEntry->bytes); memcpy(pMsg->data, serialized, len); - free(serialized); + taosMemoryFree(serialized); syncEntryDestory(pEntry); } else { @@ -99,6 +103,8 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { pMsg->prevLogTerm = preLogTerm; pMsg->commitIndex = pSyncNode->commitIndex; + syncAppendEntriesLog2("==syncNodeAppendEntriesPeers==", pMsg); + // send AppendEntries syncNodeAppendEntries(pSyncNode, pDestId, pMsg); syncAppendEntriesDestroy(pMsg); diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 8fc17ccb51ddc56623d47b35ddd73df9b655418a..287500a587feca5e0f5489de2faa2c80def38aad 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -76,10 +76,10 @@ bool syncUtilEmptyId(const SRaftId* pId) { return (pId->addr == 0 && pId->vgId = // ---- SSyncBuffer ----- void syncUtilbufBuild(SSyncBuffer* syncBuf, size_t len) { syncBuf->len = len; - syncBuf->data = malloc(syncBuf->len); + syncBuf->data = taosMemoryMalloc(syncBuf->len); } -void syncUtilbufDestroy(SSyncBuffer* syncBuf) { free(syncBuf->data); } +void syncUtilbufDestroy(SSyncBuffer* syncBuf) { taosMemoryFree(syncBuf->data); } void syncUtilbufCopy(const SSyncBuffer* src, SSyncBuffer* dest) { dest->len = src->len; @@ -88,7 +88,7 @@ void syncUtilbufCopy(const SSyncBuffer* src, SSyncBuffer* dest) { void syncUtilbufCopyDeep(const SSyncBuffer* src, SSyncBuffer* dest) { dest->len = src->len; - dest->data = malloc(dest->len); + dest->data = taosMemoryMalloc(dest->len); memcpy(dest->data, src->data, dest->len); } @@ -161,7 +161,7 @@ bool syncUtilCanPrint(char c) { } char* syncUtilprintBin(char* ptr, uint32_t len) { - char* s = malloc(len + 1); + char* s = taosMemoryMalloc(len + 1); assert(s != NULL); memset(s, 0, len + 1); memcpy(s, ptr, len); @@ -176,7 +176,7 @@ char* syncUtilprintBin(char* ptr, uint32_t len) { char* syncUtilprintBin2(char* ptr, uint32_t len) { uint32_t len2 = len * 4 + 1; - char* s = malloc(len2); + char* s = taosMemoryMalloc(len2); assert(s != NULL); memset(s, 0, len2); diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c index 69a4c48f39f9b532b106a3ba4b55871361b19a38..733dfd05b6deb88ed08df78858f358822bebbda7 100644 --- a/source/libs/sync/src/syncVoteMgr.c +++ b/source/libs/sync/src/syncVoteMgr.c @@ -23,7 +23,7 @@ static void voteGrantedClearVotes(SVotesGranted *pVotesGranted) { } SVotesGranted *voteGrantedCreate(SSyncNode *pSyncNode) { - SVotesGranted *pVotesGranted = malloc(sizeof(SVotesGranted)); + SVotesGranted *pVotesGranted = taosMemoryMalloc(sizeof(SVotesGranted)); assert(pVotesGranted != NULL); memset(pVotesGranted, 0, sizeof(SVotesGranted)); @@ -41,7 +41,7 @@ SVotesGranted *voteGrantedCreate(SSyncNode *pSyncNode) { void voteGrantedDestroy(SVotesGranted *pVotesGranted) { if (pVotesGranted != NULL) { - free(pVotesGranted); + taosMemoryFree(pVotesGranted); } } @@ -89,12 +89,12 @@ cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) { for (int i = 0; i < pVotesGranted->replicaNum; ++i) { cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas))[i])); } - int *arr = (int *)malloc(sizeof(int) * pVotesGranted->replicaNum); + int *arr = (int *)taosMemoryMalloc(sizeof(int) * pVotesGranted->replicaNum); for (int i = 0; i < pVotesGranted->replicaNum; ++i) { arr[i] = pVotesGranted->isGranted[i]; } cJSON *pIsGranted = cJSON_CreateIntArray(arr, pVotesGranted->replicaNum); - free(arr); + taosMemoryFree(arr); cJSON_AddItemToObject(pRoot, "isGranted", pIsGranted); cJSON_AddNumberToObject(pRoot, "votes", pVotesGranted->votes); @@ -126,31 +126,31 @@ void voteGrantedPrint(SVotesGranted *pObj) { char *serialized = voteGranted2Str(pObj); printf("voteGrantedPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void voteGrantedPrint2(char *s, SVotesGranted *pObj) { char *serialized = voteGranted2Str(pObj); printf("voteGrantedPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void voteGrantedLog(SVotesGranted *pObj) { char *serialized = voteGranted2Str(pObj); sTrace("voteGrantedLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void voteGrantedLog2(char *s, SVotesGranted *pObj) { char *serialized = voteGranted2Str(pObj); sTrace("voteGrantedLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // SVotesRespond ----------------------------- SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode) { - SVotesRespond *pVotesRespond = malloc(sizeof(SVotesRespond)); + SVotesRespond *pVotesRespond = taosMemoryMalloc(sizeof(SVotesRespond)); assert(pVotesRespond != NULL); memset(pVotesRespond, 0, sizeof(SVotesRespond)); @@ -164,7 +164,7 @@ SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode) { void votesRespondDestory(SVotesRespond *pVotesRespond) { if (pVotesRespond != NULL) { - free(pVotesRespond); + taosMemoryFree(pVotesRespond); } } @@ -213,7 +213,7 @@ cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) { cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesRespond->replicas))[i])); } int respondNum = 0; - int *arr = (int *)malloc(sizeof(int) * pVotesRespond->replicaNum); + int *arr = (int *)taosMemoryMalloc(sizeof(int) * pVotesRespond->replicaNum); for (int i = 0; i < pVotesRespond->replicaNum; ++i) { arr[i] = pVotesRespond->isRespond[i]; if (pVotesRespond->isRespond[i]) { @@ -221,7 +221,7 @@ cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) { } } cJSON *pIsRespond = cJSON_CreateIntArray(arr, pVotesRespond->replicaNum); - free(arr); + taosMemoryFree(arr); cJSON_AddItemToObject(pRoot, "isRespond", pIsRespond); cJSON_AddNumberToObject(pRoot, "respondNum", respondNum); @@ -248,24 +248,24 @@ void votesRespondPrint(SVotesRespond *pObj) { char *serialized = votesRespond2Str(pObj); printf("votesRespondPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void votesRespondPrint2(char *s, SVotesRespond *pObj) { char *serialized = votesRespond2Str(pObj); printf("votesRespondPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void votesRespondLog(SVotesRespond *pObj) { char *serialized = votesRespond2Str(pObj); sTrace("votesRespondLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void votesRespondLog2(char *s, SVotesRespond *pObj) { char *serialized = votesRespond2Str(pObj); sTrace("votesRespondLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index f02ba29218d3e069df4575022d22ee0745263b2d..7341565b1d526239e2b15aae2d463b294991c3da 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -29,10 +29,15 @@ add_executable(syncPingTimerTest2 "") add_executable(syncPingSelfTest "") add_executable(syncElectTest "") add_executable(syncElectTest2 "") +add_executable(syncElectTest3 "") add_executable(syncEncodeTest "") add_executable(syncWriteTest "") add_executable(syncReplicateTest "") +add_executable(syncReplicateTest2 "") +add_executable(syncReplicateTest3 "") +add_executable(syncReplicateLoadTest "") add_executable(syncRefTest "") +add_executable(syncLogStoreCheck "") target_sources(syncTest @@ -159,6 +164,10 @@ target_sources(syncElectTest2 PRIVATE "syncElectTest2.cpp" ) +target_sources(syncElectTest3 + PRIVATE + "syncElectTest3.cpp" +) target_sources(syncEncodeTest PRIVATE "syncEncodeTest.cpp" @@ -171,10 +180,26 @@ target_sources(syncReplicateTest PRIVATE "syncReplicateTest.cpp" ) +target_sources(syncReplicateTest2 + PRIVATE + "syncReplicateTest2.cpp" +) +target_sources(syncReplicateTest3 + PRIVATE + "syncReplicateTest3.cpp" +) +target_sources(syncReplicateLoadTest + PRIVATE + "syncReplicateLoadTest.cpp" +) target_sources(syncRefTest PRIVATE "syncRefTest.cpp" ) +target_sources(syncLogStoreCheck + PRIVATE + "syncLogStoreCheck.cpp" +) target_include_directories(syncTest @@ -332,6 +357,11 @@ target_include_directories(syncElectTest2 "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncElectTest3 + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_include_directories(syncEncodeTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/sync" @@ -347,11 +377,31 @@ target_include_directories(syncReplicateTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncReplicateTest2 + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncReplicateTest3 + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncReplicateLoadTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_include_directories(syncRefTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncLogStoreCheck + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -478,6 +528,10 @@ target_link_libraries(syncElectTest2 sync gtest_main ) +target_link_libraries(syncElectTest3 + sync + gtest_main +) target_link_libraries(syncEncodeTest sync gtest_main @@ -490,10 +544,26 @@ target_link_libraries(syncReplicateTest sync gtest_main ) +target_link_libraries(syncReplicateTest2 + sync + gtest_main +) +target_link_libraries(syncReplicateTest3 + sync + gtest_main +) +target_link_libraries(syncReplicateLoadTest + sync + gtest_main +) target_link_libraries(syncRefTest sync gtest_main ) +target_link_libraries(syncLogStoreCheck + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncAppendEntriesReplyTest.cpp b/source/libs/sync/test/syncAppendEntriesReplyTest.cpp index 362da67c66869a85a06b6d34398ffd5f30d0f165..72aeb155e7eb0a0aec376802e57849b1410a3ccc 100644 --- a/source/libs/sync/test/syncAppendEntriesReplyTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesReplyTest.cpp @@ -34,14 +34,14 @@ void test1() { void test2() { SyncAppendEntriesReply *pMsg = createMsg(); uint32_t len = pMsg->bytes; - char * serialized = (char *)malloc(len); + char * serialized = (char *)taosMemoryMalloc(len); syncAppendEntriesReplySerialize(pMsg, serialized, len); SyncAppendEntriesReply *pMsg2 = syncAppendEntriesReplyBuild(); syncAppendEntriesReplyDeserialize(serialized, len, pMsg2); syncAppendEntriesReplyPrint2((char *)"test2: syncAppendEntriesReplySerialize -> syncAppendEntriesReplyDeserialize ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncAppendEntriesReplyDestroy(pMsg); syncAppendEntriesReplyDestroy(pMsg2); } @@ -54,7 +54,7 @@ void test3() { syncAppendEntriesReplyPrint2((char *)"test3: syncAppendEntriesReplySerialize3 -> syncAppendEntriesReplyDeserialize2 ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncAppendEntriesReplyDestroy(pMsg); syncAppendEntriesReplyDestroy(pMsg2); } diff --git a/source/libs/sync/test/syncAppendEntriesTest.cpp b/source/libs/sync/test/syncAppendEntriesTest.cpp index 687d9bcb945dbbcddcf0d543f67ea578c5bc4617..69a0aee9a8b389a322669adee70459d07e7c3f11 100644 --- a/source/libs/sync/test/syncAppendEntriesTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesTest.cpp @@ -36,13 +36,13 @@ void test1() { void test2() { SyncAppendEntries *pMsg = createMsg(); uint32_t len = pMsg->bytes; - char * serialized = (char *)malloc(len); + char * serialized = (char *)taosMemoryMalloc(len); syncAppendEntriesSerialize(pMsg, serialized, len); SyncAppendEntries *pMsg2 = syncAppendEntriesBuild(pMsg->dataLen); syncAppendEntriesDeserialize(serialized, len, pMsg2); syncAppendEntriesPrint2((char *)"test2: syncAppendEntriesSerialize -> syncAppendEntriesDeserialize ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncAppendEntriesDestroy(pMsg); syncAppendEntriesDestroy(pMsg2); } @@ -54,7 +54,7 @@ void test3() { SyncAppendEntries *pMsg2 = syncAppendEntriesDeserialize2(serialized, len); syncAppendEntriesPrint2((char *)"test3: syncAppendEntriesSerialize3 -> syncAppendEntriesDeserialize2 ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncAppendEntriesDestroy(pMsg); syncAppendEntriesDestroy(pMsg2); } @@ -63,7 +63,7 @@ void test4() { SyncAppendEntries *pMsg = createMsg(); SRpcMsg rpcMsg; syncAppendEntries2RpcMsg(pMsg, &rpcMsg); - SyncAppendEntries *pMsg2 = (SyncAppendEntries *)malloc(rpcMsg.contLen); + SyncAppendEntries *pMsg2 = (SyncAppendEntries *)taosMemoryMalloc(rpcMsg.contLen); syncAppendEntriesFromRpcMsg(&rpcMsg, pMsg2); syncAppendEntriesPrint2((char *)"test4: syncAppendEntries2RpcMsg -> syncAppendEntriesFromRpcMsg ", pMsg2); diff --git a/source/libs/sync/test/syncClientRequestTest.cpp b/source/libs/sync/test/syncClientRequestTest.cpp index 6323f53a0354897e5b2f6525573e4eeade592014..f22478d53878f88cab0461f5af619f8f20070b51 100644 --- a/source/libs/sync/test/syncClientRequestTest.cpp +++ b/source/libs/sync/test/syncClientRequestTest.cpp @@ -34,13 +34,13 @@ void test1() { void test2() { SyncClientRequest *pMsg = createMsg(); uint32_t len = pMsg->bytes; - char * serialized = (char *)malloc(len); + char * serialized = (char *)taosMemoryMalloc(len); syncClientRequestSerialize(pMsg, serialized, len); SyncClientRequest *pMsg2 = syncClientRequestBuild(pMsg->dataLen); syncClientRequestDeserialize(serialized, len, pMsg2); syncClientRequestPrint2((char *)"test2: syncClientRequestSerialize -> syncClientRequestDeserialize ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncClientRequestDestroy(pMsg); syncClientRequestDestroy(pMsg2); } @@ -52,7 +52,7 @@ void test3() { SyncClientRequest *pMsg2 = syncClientRequestDeserialize2(serialized, len); syncClientRequestPrint2((char *)"test3: syncClientRequestSerialize3 -> syncClientRequestDeserialize2 ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncClientRequestDestroy(pMsg); syncClientRequestDestroy(pMsg2); } @@ -61,7 +61,7 @@ void test4() { SyncClientRequest *pMsg = createMsg(); SRpcMsg rpcMsg; syncClientRequest2RpcMsg(pMsg, &rpcMsg); - SyncClientRequest *pMsg2 = (SyncClientRequest *)malloc(rpcMsg.contLen); + SyncClientRequest *pMsg2 = (SyncClientRequest *)taosMemoryMalloc(rpcMsg.contLen); syncClientRequestFromRpcMsg(&rpcMsg, pMsg2); syncClientRequestPrint2((char *)"test4: syncClientRequest2RpcMsg -> syncClientRequestFromRpcMsg ", pMsg2); diff --git a/source/libs/sync/test/syncElectTest.cpp b/source/libs/sync/test/syncElectTest.cpp index 99cfe75867db01ac99ecb0209e950aa59029aaf8..251a4b538fe24574325136cb36f6f4037bdd8078 100644 --- a/source/libs/sync/test/syncElectTest.cpp +++ b/source/libs/sync/test/syncElectTest.cpp @@ -88,7 +88,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncElectTest2.cpp b/source/libs/sync/test/syncElectTest2.cpp index 8463ffd448aa62cb999642b5749c7b8ee4325cbf..709d9d8580f91c6221456019a9c6a67a8eeb8736 100644 --- a/source/libs/sync/test/syncElectTest2.cpp +++ b/source/libs/sync/test/syncElectTest2.cpp @@ -91,7 +91,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncElectTest3.cpp b/source/libs/sync/test/syncElectTest3.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e36a2d0fb1c14669fe10d90ca0ace4aa8e877a9f --- /dev/null +++ b/source/libs/sync/test/syncElectTest3.cpp @@ -0,0 +1,138 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "tref.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 3; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; +SWal* pWal; + +int64_t syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "./elect3_test_%d", myIndex); + + int code = walInit(); + assert(code == 0); + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = syncInfo.vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + + char tmpdir[128]; + snprintf(tmpdir, sizeof(tmpdir), "./elect3_test_wal_%d", myIndex); + pWal = walOpen(tmpdir, &walCfg); + assert(pWal != NULL); + + syncInfo.pWal = pWal; + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + int64_t rid = syncStart(&syncInfo); + assert(rid > 0); + + SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + pSyncNode->hbBaseLine = 500; + pSyncNode->electBaseLine = 1500; + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->pSyncNode = pSyncNode; + + syncNodeRelease(pSyncNode); + + return rid; +} + +void initRaftId(SSyncNode* pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + taosMemoryFree(s); + } +} + +int main(int argc, char** argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncInit(); + assert(ret == 0); + + int64_t rid = syncNodeInit(); + assert(rid > 0); + + SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + syncNodePrint2((char*)"", pSyncNode); + initRaftId(pSyncNode); + + //--------------------------- + while (1) { + sTrace( + "elect sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d", + pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS); + taosMsleep(1000); + } + + syncNodeRelease(pSyncNode); + + return 0; +} diff --git a/source/libs/sync/test/syncEncodeTest.cpp b/source/libs/sync/test/syncEncodeTest.cpp index e554858072bf6a3f71c8ec1c185ead2274a0b852..b1ba95dc3e46d5ecfe08afa007bba860cb33eab2 100644 --- a/source/libs/sync/test/syncEncodeTest.cpp +++ b/source/libs/sync/test/syncEncodeTest.cpp @@ -87,16 +87,16 @@ void initRaftId(SSyncNode *pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char *s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } SRpcMsg *step0() { - SRpcMsg *pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + SRpcMsg *pMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); memset(pMsg, 0, sizeof(SRpcMsg)); pMsg->msgType = 9999; pMsg->contLen = 32; - pMsg->pCont = malloc(pMsg->contLen); + pMsg->pCont = taosMemoryMalloc(pMsg->contLen); snprintf((char *)(pMsg->pCont), pMsg->contLen, "hello, world"); return pMsg; } @@ -107,7 +107,7 @@ SyncClientRequest *step1(const SRpcMsg *pMsg) { } SRpcMsg *step2(const SyncClientRequest *pMsg) { - SRpcMsg *pRetMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + SRpcMsg *pRetMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); syncClientRequest2RpcMsg(pMsg, pRetMsg); return pRetMsg; } @@ -133,7 +133,7 @@ SSyncRaftEntry *step6(const char *pMsg, uint32_t len) { } SRpcMsg *step7(const SSyncRaftEntry *pMsg) { - SRpcMsg *pRetMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + SRpcMsg *pRetMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); syncEntry2OriginalRpc(pMsg, pRetMsg); return pRetMsg; } @@ -190,7 +190,7 @@ int main(int argc, char **argv) { char * pMsg5 = step5(pMsg4, &len); char * s = syncUtilprintBin(pMsg5, len); printf("==step5== [%s] \n", s); - free(s); + taosMemoryFree(s); // step6 SSyncRaftEntry *pMsg6 = step6(pMsg5, len); diff --git a/source/libs/sync/test/syncEnqTest.cpp b/source/libs/sync/test/syncEnqTest.cpp index 57315f40ec31a9148727ac1c4a822c267b119b36..48cc7caf2f433d3268c8a6b8e7b2ad44231f0d0a 100644 --- a/source/libs/sync/test/syncEnqTest.cpp +++ b/source/libs/sync/test/syncEnqTest.cpp @@ -66,7 +66,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncEntryTest.cpp b/source/libs/sync/test/syncEntryTest.cpp index e54daaa79a052504fd8cb8bbbb7f59baec9e6385..e8427e81685a22e293fab61ac34071adb7b3b54f 100644 --- a/source/libs/sync/test/syncEntryTest.cpp +++ b/source/libs/sync/test/syncEntryTest.cpp @@ -46,6 +46,20 @@ void test2() { } void test3() { + SyncClientRequest* pSyncMsg = syncClientRequestBuild(10); + pSyncMsg->originalRpcType = 33; + pSyncMsg->seqNum = 11; + pSyncMsg->isWeak = 1; + strcpy(pSyncMsg->data, "test3"); + + SSyncRaftEntry* pEntry = syncEntryBuild3(pSyncMsg, 100, 200, SYNC_RAFT_ENTRY_NOOP); + syncEntryPrint(pEntry); + + syncClientRequestDestroy(pSyncMsg); + syncEntryDestory(pEntry); +} + +void test4() { SSyncRaftEntry* pEntry = syncEntryBuild(10); assert(pEntry != NULL); pEntry->msgType = 11; @@ -54,7 +68,8 @@ void test3() { pEntry->isWeak = true; pEntry->term = 44; pEntry->index = 55; - strcpy(pEntry->data, "test3"); + pEntry->entryType = SYNC_RAFT_ENTRY_CONFIG; + strcpy(pEntry->data, "test4"); syncEntryPrint(pEntry); uint32_t len; @@ -63,7 +78,7 @@ void test3() { SSyncRaftEntry* pEntry2 = syncEntryDeserialize(serialized, len); syncEntryPrint(pEntry2); - free(serialized); + taosMemoryFree(serialized); syncEntryDestory(pEntry2); syncEntryDestory(pEntry); } @@ -76,6 +91,7 @@ int main(int argc, char** argv) { test1(); test2(); test3(); + test4(); return 0; } diff --git a/source/libs/sync/test/syncIOSendMsgTest.cpp b/source/libs/sync/test/syncIOSendMsgTest.cpp index 0fc3ebfe4cdc5e4417f34c905c039ac5c31fc707..913e57e59abfcf7ded83c9061d33026243dea2fb 100644 --- a/source/libs/sync/test/syncIOSendMsgTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgTest.cpp @@ -66,7 +66,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncIndexMgrTest.cpp b/source/libs/sync/test/syncIndexMgrTest.cpp index 6bad8f09cf3f5912f0ef4ba71316dd3654af91fd..319ea3e15a176f5f77df0cc88fbd8d953d7678f4 100644 --- a/source/libs/sync/test/syncIndexMgrTest.cpp +++ b/source/libs/sync/test/syncIndexMgrTest.cpp @@ -68,7 +68,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } @@ -93,7 +93,7 @@ int main(int argc, char** argv) { char* serialized = syncNode2Str(pSyncNode); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); initRaftId(pSyncNode); @@ -105,7 +105,7 @@ int main(int argc, char** argv) { char* serialized = syncIndexMgr2Str(pSyncIndexMgr); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } syncIndexMgrSetIndex(pSyncIndexMgr, &ids[0], 100); @@ -117,7 +117,7 @@ int main(int argc, char** argv) { char* serialized = syncIndexMgr2Str(pSyncIndexMgr); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } printf("---------------------------------------\n"); @@ -132,7 +132,7 @@ int main(int argc, char** argv) { char* serialized = syncIndexMgr2Str(pSyncIndexMgr); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } syncIndexMgrDestroy(pSyncIndexMgr); diff --git a/source/libs/sync/test/syncInitTest.cpp b/source/libs/sync/test/syncInitTest.cpp index a3e5f41c85765e771cc8ab36efd663117f4cecab..48b5488e416c442ffacff6e0884efefabffbb6ec 100644 --- a/source/libs/sync/test/syncInitTest.cpp +++ b/source/libs/sync/test/syncInitTest.cpp @@ -65,7 +65,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncLogStoreCheck.cpp b/source/libs/sync/test/syncLogStoreCheck.cpp new file mode 100644 index 0000000000000000000000000000000000000000..85e39a61c56c47f7f90d6465c5f70f5a508f5ea8 --- /dev/null +++ b/source/libs/sync/test/syncLogStoreCheck.cpp @@ -0,0 +1,105 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 1; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; +SWal* pWal; +SSyncNode* pSyncNode; + +SSyncNode* syncNodeInit(const char* path) { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./log_check"); + + int code = walInit(); + assert(code == 0); + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = syncInfo.vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + pWal = walOpen(path, &walCfg); + assert(pWal != NULL); + + syncInfo.pWal = pWal; + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->pSyncNode = pSyncNode; + + return pSyncNode; +} + +SSyncNode* logStoreCheck(const char* path) { return syncNodeInit(path); } + +int main(int argc, char** argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + pSyncNode = logStoreCheck(argv[1]); + assert(pSyncNode != NULL); + + logStorePrint2((char*)"logStoreCheck", pSyncNode->pLogStore); + + return 0; +} diff --git a/source/libs/sync/test/syncLogStoreTest.cpp b/source/libs/sync/test/syncLogStoreTest.cpp index c1cb66f382574f042bf910308a5da5e2d1a1b0b1..a5adba9a8861b7f12f8bc6259e7f466c2b3b27db 100644 --- a/source/libs/sync/test/syncLogStoreTest.cpp +++ b/source/libs/sync/test/syncLogStoreTest.cpp @@ -116,7 +116,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncPingReplyTest.cpp b/source/libs/sync/test/syncPingReplyTest.cpp index 8e1448e7818c8d41c6b880208c2f4fe78ecb4035..40592eab6f7935fb048c27455c1aa4aace077c17 100644 --- a/source/libs/sync/test/syncPingReplyTest.cpp +++ b/source/libs/sync/test/syncPingReplyTest.cpp @@ -33,13 +33,13 @@ void test1() { void test2() { SyncPingReply *pMsg = createMsg(); uint32_t len = pMsg->bytes; - char * serialized = (char *)malloc(len); + char * serialized = (char *)taosMemoryMalloc(len); syncPingReplySerialize(pMsg, serialized, len); SyncPingReply *pMsg2 = syncPingReplyBuild(pMsg->dataLen); syncPingReplyDeserialize(serialized, len, pMsg2); syncPingReplyPrint2((char *)"test2: syncPingReplySerialize -> syncPingReplyDeserialize ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncPingReplyDestroy(pMsg); syncPingReplyDestroy(pMsg2); } @@ -51,7 +51,7 @@ void test3() { SyncPingReply *pMsg2 = syncPingReplyDeserialize2(serialized, len); syncPingReplyPrint2((char *)"test3: syncPingReplySerialize3 -> syncPingReplyDeserialize2 ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncPingReplyDestroy(pMsg); syncPingReplyDestroy(pMsg2); } @@ -60,7 +60,7 @@ void test4() { SyncPingReply *pMsg = createMsg(); SRpcMsg rpcMsg; syncPingReply2RpcMsg(pMsg, &rpcMsg); - SyncPingReply *pMsg2 = (SyncPingReply *)malloc(rpcMsg.contLen); + SyncPingReply *pMsg2 = (SyncPingReply *)taosMemoryMalloc(rpcMsg.contLen); syncPingReplyFromRpcMsg(&rpcMsg, pMsg2); syncPingReplyPrint2((char *)"test4: syncPingReply2RpcMsg -> syncPingReplyFromRpcMsg ", pMsg2); diff --git a/source/libs/sync/test/syncPingSelfTest.cpp b/source/libs/sync/test/syncPingSelfTest.cpp index 05e4d99cb0cb44388f82bf154ff28769d5766162..677518c6ac76b3d16f80bfdefb4361655f245ba4 100644 --- a/source/libs/sync/test/syncPingSelfTest.cpp +++ b/source/libs/sync/test/syncPingSelfTest.cpp @@ -65,7 +65,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp index 83394b0e77404b32ef665643fc85cdf6f8a602b6..eb774d77c31166071079ad4b39fc30f64743fce3 100644 --- a/source/libs/sync/test/syncPingTest.cpp +++ b/source/libs/sync/test/syncPingTest.cpp @@ -33,13 +33,13 @@ void test1() { void test2() { SyncPing *pMsg = createMsg(); uint32_t len = pMsg->bytes; - char * serialized = (char *)malloc(len); + char * serialized = (char *)taosMemoryMalloc(len); syncPingSerialize(pMsg, serialized, len); SyncPing *pMsg2 = syncPingBuild(pMsg->dataLen); syncPingDeserialize(serialized, len, pMsg2); syncPingPrint2((char *)"test2: syncPingSerialize -> syncPingDeserialize ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncPingDestroy(pMsg); syncPingDestroy(pMsg2); } @@ -51,7 +51,7 @@ void test3() { SyncPing *pMsg2 = syncPingDeserialize2(serialized, len); syncPingPrint2((char *)"test3: syncPingSerialize3 -> syncPingDeserialize2 ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncPingDestroy(pMsg); syncPingDestroy(pMsg2); } @@ -60,7 +60,7 @@ void test4() { SyncPing *pMsg = createMsg(); SRpcMsg rpcMsg; syncPing2RpcMsg(pMsg, &rpcMsg); - SyncPing *pMsg2 = (SyncPing *)malloc(rpcMsg.contLen); + SyncPing *pMsg2 = (SyncPing *)taosMemoryMalloc(rpcMsg.contLen); syncPingFromRpcMsg(&rpcMsg, pMsg2); syncPingPrint2((char *)"test4: syncPing2RpcMsg -> syncPingFromRpcMsg ", pMsg2); diff --git a/source/libs/sync/test/syncPingTimerTest.cpp b/source/libs/sync/test/syncPingTimerTest.cpp index 20d4a9ce589bad5b399f727729a9fe7f21c0d94f..3c5e76ca147aeb2b438bc2a5afb0e90417cf0cc9 100644 --- a/source/libs/sync/test/syncPingTimerTest.cpp +++ b/source/libs/sync/test/syncPingTimerTest.cpp @@ -65,7 +65,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncPingTimerTest2.cpp b/source/libs/sync/test/syncPingTimerTest2.cpp index 2a041f3f5ddf4a530127699a0e64645bfd40b1f4..554f67d3658480c34793e3c3663d7caf84142688 100644 --- a/source/libs/sync/test/syncPingTimerTest2.cpp +++ b/source/libs/sync/test/syncPingTimerTest2.cpp @@ -65,7 +65,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncRaftStoreTest.cpp b/source/libs/sync/test/syncRaftStoreTest.cpp index 688802625ac42203b7f52a459b7440ff3324f8e8..460ff90f4f752b76182fc635fe62505ca2e37f66 100644 --- a/source/libs/sync/test/syncRaftStoreTest.cpp +++ b/source/libs/sync/test/syncRaftStoreTest.cpp @@ -25,7 +25,7 @@ void initRaftId() { ids[i].vgId = 1234; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncRefTest.cpp b/source/libs/sync/test/syncRefTest.cpp index 8e9adbbc151168abe6558a3172f51a17e00c5d60..96062b1a91b235ce24a11cfb91a0b13ac14c5c23 100644 --- a/source/libs/sync/test/syncRefTest.cpp +++ b/source/libs/sync/test/syncRefTest.cpp @@ -33,7 +33,7 @@ typedef struct SyncObj { static void syncFreeObj(void *param) { SyncObj *pObj = (SyncObj *)param; printf("syncFreeObj name:%s rid:%ld \n", pObj->name, pObj->rid); - free(pObj); + taosMemoryFree(pObj); } int32_t init() { @@ -54,7 +54,7 @@ void cleanup() { } int64_t start() { - SyncObj *pObj = (SyncObj *)malloc(sizeof(SyncObj)); + SyncObj *pObj = (SyncObj *)taosMemoryMalloc(sizeof(SyncObj)); assert(pObj != NULL); pObj->data = &g; diff --git a/source/libs/sync/test/syncReplicateLoadTest.cpp b/source/libs/sync/test/syncReplicateLoadTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0841083b4a5cb00db2f4f0b3cc758c991fd72364 --- /dev/null +++ b/source/libs/sync/test/syncReplicateLoadTest.cpp @@ -0,0 +1,189 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncRaftEntry.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 3; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM *pFsm; +SWal * pWal; + +void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, index, isWeak, + code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void initFsm() { + pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM)); + pFsm->FpCommitCb = CommitCb; + pFsm->FpPreCommitCb = PreCommitCb; + pFsm->FpRollBackCb = RollBackCb; +} + +int64_t syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "./replicate2_test_%d", myIndex); + + int code = walInit(); + assert(code == 0); + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = syncInfo.vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + + char tmpdir[128]; + snprintf(tmpdir, sizeof(tmpdir), "./replicate2_test_wal_%d", myIndex); + pWal = walOpen(tmpdir, &walCfg); + assert(pWal != NULL); + + syncInfo.pWal = pWal; + + SSyncCfg *pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + int64_t rid = syncStart(&syncInfo); + assert(rid > 0); + + SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + // pSyncNode->hbBaseLine = 500; + // pSyncNode->electBaseLine = 1500; + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; + gSyncIO->pSyncNode = pSyncNode; + + syncNodeRelease(pSyncNode); + + return rid; +} + +void initRaftId(SSyncNode *pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char *s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + taosMemoryFree(s); + } +} + +SRpcMsg *step0(int i) { + SRpcMsg *pMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); + memset(pMsg, 0, sizeof(SRpcMsg)); + pMsg->msgType = 9999; + pMsg->contLen = 128; + pMsg->pCont = taosMemoryMalloc(pMsg->contLen); + snprintf((char *)(pMsg->pCont), pMsg->contLen, "value-%u-%d", ports[myIndex], i); + return pMsg; +} + +SyncClientRequest *step1(const SRpcMsg *pMsg) { + SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true); + return pRetMsg; +} + +int main(int argc, char **argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + void logTest(); + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char *)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + initFsm(); + + ret = syncInit(); + assert(ret == 0); + + int64_t rid = syncNodeInit(); + assert(rid > 0); + + SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + syncNodePrint2((char *)"", pSyncNode); + initRaftId(pSyncNode); + + // only load ... + + while (1) { + sTrace( + "replicate sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " + "electTimerMS:%d, commitIndex:%ld", + pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS, + pSyncNode->commitIndex); + taosMsleep(1000); + } + + return 0; +} diff --git a/source/libs/sync/test/syncReplicateTest.cpp b/source/libs/sync/test/syncReplicateTest.cpp index 4d6e6f3a25f92f5e5cc33b65b249175cfb212f65..9783bac7e551076bc71a33194af24de530883c6b 100644 --- a/source/libs/sync/test/syncReplicateTest.cpp +++ b/source/libs/sync/test/syncReplicateTest.cpp @@ -28,23 +28,33 @@ SSyncFSM * pFsm; SWal * pWal; SSyncNode *gSyncNode; -void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) { - printf("==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code); - syncRpcMsgPrint2((char *)"==CommitCb==", (SRpcMsg *)pBuf); +void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); } -void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) { - printf("==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code); - syncRpcMsgPrint2((char *)"==PreCommitCb==", (SRpcMsg *)pBuf); +void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, index, isWeak, + code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); } -void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) { - printf("==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code); - syncRpcMsgPrint2((char *)"==RollBackCb==", (SRpcMsg *)pBuf); +void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); } void initFsm() { - pFsm = (SSyncFSM *)malloc(sizeof(SSyncFSM)); + pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM)); pFsm->FpCommitCb = CommitCb; pFsm->FpPreCommitCb = PreCommitCb; pFsm->FpRollBackCb = RollBackCb; @@ -111,16 +121,16 @@ void initRaftId(SSyncNode *pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char *s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } SRpcMsg *step0(int i) { - SRpcMsg *pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + SRpcMsg *pMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); memset(pMsg, 0, sizeof(SRpcMsg)); pMsg->msgType = 9999; pMsg->contLen = 128; - pMsg->pCont = malloc(pMsg->contLen); + pMsg->pCont = taosMemoryMalloc(pMsg->contLen); snprintf((char *)(pMsg->pCont), pMsg->contLen, "value-%u-%d", ports[myIndex], i); return pMsg; } @@ -173,18 +183,20 @@ int main(int argc, char **argv) { taosMsleep(1000); sTrace( - "replicate sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " - "electTimerMS:%d", + "syncPropose sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " + "electTimerMS:%d, commitIndex:%ld", gSyncNode->state, syncUtilState2String(gSyncNode->state), gSyncNode->pRaftStore->currentTerm, - gSyncNode->electTimerLogicClock, gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS); + gSyncNode->electTimerLogicClock, gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS, + gSyncNode->commitIndex); } while (1) { sTrace( "replicate sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " - "electTimerMS:%d", + "electTimerMS:%d, commitIndex:%ld", gSyncNode->state, syncUtilState2String(gSyncNode->state), gSyncNode->pRaftStore->currentTerm, - gSyncNode->electTimerLogicClock, gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS); + gSyncNode->electTimerLogicClock, gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS, + gSyncNode->commitIndex); taosMsleep(1000); } diff --git a/source/libs/sync/test/syncReplicateTest2.cpp b/source/libs/sync/test/syncReplicateTest2.cpp new file mode 100644 index 0000000000000000000000000000000000000000..430df7eebdbc7333d9240bdc4e4a8d13ccb18937 --- /dev/null +++ b/source/libs/sync/test/syncReplicateTest2.cpp @@ -0,0 +1,205 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncRaftEntry.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 3; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM *pFsm; +SWal * pWal; + +void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, index, isWeak, + code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void initFsm() { + pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM)); + pFsm->FpCommitCb = CommitCb; + pFsm->FpPreCommitCb = PreCommitCb; + pFsm->FpRollBackCb = RollBackCb; +} + +int64_t syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "./replicate2_test_%d", myIndex); + + int code = walInit(); + assert(code == 0); + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = syncInfo.vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + + char tmpdir[128]; + snprintf(tmpdir, sizeof(tmpdir), "./replicate2_test_wal_%d", myIndex); + pWal = walOpen(tmpdir, &walCfg); + assert(pWal != NULL); + + syncInfo.pWal = pWal; + + SSyncCfg *pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + int64_t rid = syncStart(&syncInfo); + assert(rid > 0); + + SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + // pSyncNode->hbBaseLine = 500; + // pSyncNode->electBaseLine = 1500; + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; + gSyncIO->pSyncNode = pSyncNode; + + syncNodeRelease(pSyncNode); + + return rid; +} + +void initRaftId(SSyncNode *pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char *s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + taosMemoryFree(s); + } +} + +SRpcMsg *step0(int i) { + SRpcMsg *pMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); + memset(pMsg, 0, sizeof(SRpcMsg)); + pMsg->msgType = 9999; + pMsg->contLen = 128; + pMsg->pCont = taosMemoryMalloc(pMsg->contLen); + snprintf((char *)(pMsg->pCont), pMsg->contLen, "value-%u-%d", ports[myIndex], i); + return pMsg; +} + +SyncClientRequest *step1(const SRpcMsg *pMsg) { + SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true); + return pRetMsg; +} + +int main(int argc, char **argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + void logTest(); + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char *)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + initFsm(); + + ret = syncInit(); + assert(ret == 0); + + int64_t rid = syncNodeInit(); + assert(rid > 0); + + SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + syncNodePrint2((char *)"", pSyncNode); + initRaftId(pSyncNode); + + for (int i = 0; i < 30; ++i) { + // step0 + SRpcMsg *pMsg0 = step0(i); + syncRpcMsgPrint2((char *)"==step0==", pMsg0); + + syncPropose(rid, pMsg0, true); + taosMsleep(1000); + + taosMemoryFree(pMsg0); + + sTrace( + "syncPropose sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " + "electTimerMS:%d, commitIndex:%ld", + pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS, + pSyncNode->commitIndex); + } + + while (1) { + sTrace( + "replicate sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " + "electTimerMS:%d, commitIndex:%ld", + pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS, + pSyncNode->commitIndex); + taosMsleep(1000); + } + + return 0; +} diff --git a/source/libs/sync/test/syncReplicateTest3.cpp b/source/libs/sync/test/syncReplicateTest3.cpp new file mode 100644 index 0000000000000000000000000000000000000000..792b9c94cf022583e2cbee5185e8ef4dd7eefef4 --- /dev/null +++ b/source/libs/sync/test/syncReplicateTest3.cpp @@ -0,0 +1,217 @@ +#define ALLOW_FORBID_FUNC + +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncRaftEntry.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 3; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM *pFsm; +SWal * pWal; + +void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, index, isWeak, + code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void initFsm() { + pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM)); + pFsm->FpCommitCb = CommitCb; + pFsm->FpPreCommitCb = PreCommitCb; + pFsm->FpRollBackCb = RollBackCb; +} + +int64_t syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "./replicate2_test_%d", myIndex); + + int code = walInit(); + assert(code == 0); + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = syncInfo.vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + + char tmpdir[128]; + snprintf(tmpdir, sizeof(tmpdir), "./replicate2_test_wal_%d", myIndex); + pWal = walOpen(tmpdir, &walCfg); + assert(pWal != NULL); + + syncInfo.pWal = pWal; + + SSyncCfg *pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + int64_t rid = syncStart(&syncInfo); + assert(rid > 0); + + SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + // pSyncNode->hbBaseLine = 500; + // pSyncNode->electBaseLine = 1500; + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; + gSyncIO->pSyncNode = pSyncNode; + + syncNodeRelease(pSyncNode); + + return rid; +} + +void initRaftId(SSyncNode *pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char *s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + taosMemoryFree(s); + } +} + +SRpcMsg *step0(int i) { + SRpcMsg *pMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); + memset(pMsg, 0, sizeof(SRpcMsg)); + pMsg->msgType = 9999; + pMsg->contLen = 128; + pMsg->pCont = taosMemoryMalloc(pMsg->contLen); + snprintf((char *)(pMsg->pCont), pMsg->contLen, "value-%u-%d", ports[myIndex], i); + return pMsg; +} + +SyncClientRequest *step1(const SRpcMsg *pMsg) { + SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true); + return pRetMsg; +} + +int main(int argc, char **argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + void logTest(); + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int recordCount = 100; + if (argc >= 3) { + recordCount = atoi(argv[2]); + } + + int sleepMS = 10; + if (argc >= 4) { + sleepMS = atoi(argv[3]); + } + + int32_t ret = syncIOStart((char *)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + initFsm(); + + ret = syncInit(); + assert(ret == 0); + + int64_t rid = syncNodeInit(); + assert(rid > 0); + + SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + syncNodePrint2((char *)"", pSyncNode); + initRaftId(pSyncNode); + + for (int i = 0; i < recordCount; ++i) { + // step0 + SRpcMsg *pMsg0 = step0(i); + syncRpcMsgPrint2((char *)"==step0==", pMsg0); + + syncPropose(rid, pMsg0, true); + taosMsleep(sleepMS); + + taosMemoryFree(pMsg0); + + sTrace( + "syncPropose sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " + "electTimerMS:%d, commitIndex:%ld", + pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS, + pSyncNode->commitIndex); + } + + while (1) { + sTrace( + "replicate sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " + "electTimerMS:%d, commitIndex:%ld", + pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS, + pSyncNode->commitIndex); + taosMsleep(1000); + } + + return 0; +} diff --git a/source/libs/sync/test/syncRequestVoteReplyTest.cpp b/source/libs/sync/test/syncRequestVoteReplyTest.cpp index 2bce3e4cd6b00e077e6e5abc54f54a94b357995e..9e0ebee31318be2baa71e3b927dc2adb4269260f 100644 --- a/source/libs/sync/test/syncRequestVoteReplyTest.cpp +++ b/source/libs/sync/test/syncRequestVoteReplyTest.cpp @@ -34,13 +34,13 @@ void test1() { void test2() { SyncRequestVoteReply *pMsg = createMsg(); uint32_t len = pMsg->bytes; - char * serialized = (char *)malloc(len); + char * serialized = (char *)taosMemoryMalloc(len); syncRequestVoteReplySerialize(pMsg, serialized, len); SyncRequestVoteReply *pMsg2 = syncRequestVoteReplyBuild(); syncRequestVoteReplyDeserialize(serialized, len, pMsg2); syncRequestVoteReplyPrint2((char *)"test2: syncRequestVoteReplySerialize -> syncRequestVoteReplyDeserialize ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncRequestVoteReplyDestroy(pMsg); syncRequestVoteReplyDestroy(pMsg2); } @@ -53,7 +53,7 @@ void test3() { syncRequestVoteReplyPrint2((char *)"test3: syncRequestVoteReplySerialize3 -> syncRequestVoteReplyDeserialize2 ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncRequestVoteReplyDestroy(pMsg); syncRequestVoteReplyDestroy(pMsg2); } diff --git a/source/libs/sync/test/syncRequestVoteTest.cpp b/source/libs/sync/test/syncRequestVoteTest.cpp index 22f47046dea3a8d98e00e5b3fa28d89dfdfd7a04..6b2120e124f72dd64d32c9e6ed3e2b8edabd9df1 100644 --- a/source/libs/sync/test/syncRequestVoteTest.cpp +++ b/source/libs/sync/test/syncRequestVoteTest.cpp @@ -35,13 +35,13 @@ void test1() { void test2() { SyncRequestVote *pMsg = createMsg(); uint32_t len = pMsg->bytes; - char * serialized = (char *)malloc(len); + char * serialized = (char *)taosMemoryMalloc(len); syncRequestVoteSerialize(pMsg, serialized, len); SyncRequestVote *pMsg2 = syncRequestVoteBuild(); syncRequestVoteDeserialize(serialized, len, pMsg2); syncRequestVotePrint2((char *)"test2: syncRequestVoteSerialize -> syncRequestVoteDeserialize ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncRequestVoteDestroy(pMsg); syncRequestVoteDestroy(pMsg2); } @@ -53,7 +53,7 @@ void test3() { SyncRequestVote *pMsg2 = syncRequestVoteDeserialize2(serialized, len); syncRequestVotePrint2((char *)"test3: syncRequestVoteSerialize3 -> syncRequestVoteDeserialize2 ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncRequestVoteDestroy(pMsg); syncRequestVoteDestroy(pMsg2); } diff --git a/source/libs/sync/test/syncTimeoutTest.cpp b/source/libs/sync/test/syncTimeoutTest.cpp index 3f46ab5c7cebc4283ba8edfcbc76315d1a360023..ab36f42d0d00457cb9d6ca602d2623b9c1296971 100644 --- a/source/libs/sync/test/syncTimeoutTest.cpp +++ b/source/libs/sync/test/syncTimeoutTest.cpp @@ -30,13 +30,13 @@ void test1() { void test2() { SyncTimeout *pMsg = createMsg(); uint32_t len = pMsg->bytes; - char * serialized = (char *)malloc(len); + char * serialized = (char *)taosMemoryMalloc(len); syncTimeoutSerialize(pMsg, serialized, len); SyncTimeout *pMsg2 = syncTimeoutBuild(); syncTimeoutDeserialize(serialized, len, pMsg2); syncTimeoutPrint2((char *)"test2: syncTimeoutSerialize -> syncTimeoutDeserialize ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncTimeoutDestroy(pMsg); syncTimeoutDestroy(pMsg2); } @@ -48,7 +48,7 @@ void test3() { SyncTimeout *pMsg2 = syncTimeoutDeserialize2(serialized, len); syncTimeoutPrint2((char *)"test3: syncTimeoutSerialize3 -> syncTimeoutDeserialize2 ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncTimeoutDestroy(pMsg); syncTimeoutDestroy(pMsg2); } @@ -57,7 +57,7 @@ void test4() { SyncTimeout *pMsg = createMsg(); SRpcMsg rpcMsg; syncTimeout2RpcMsg(pMsg, &rpcMsg); - SyncTimeout *pMsg2 = (SyncTimeout *)malloc(rpcMsg.contLen); + SyncTimeout *pMsg2 = (SyncTimeout *)taosMemoryMalloc(rpcMsg.contLen); syncTimeoutFromRpcMsg(&rpcMsg, pMsg2); syncTimeoutPrint2((char *)"test4: syncTimeout2RpcMsg -> syncTimeoutFromRpcMsg ", pMsg2); diff --git a/source/libs/sync/test/syncVotesGrantedTest.cpp b/source/libs/sync/test/syncVotesGrantedTest.cpp index 588eb32ffd75d648879f923e619cd624aab5ba68..6164aed3513128daf41bcade0e68c4f1c1e94956 100644 --- a/source/libs/sync/test/syncVotesGrantedTest.cpp +++ b/source/libs/sync/test/syncVotesGrantedTest.cpp @@ -67,7 +67,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } @@ -92,7 +92,7 @@ int main(int argc, char** argv) { char* serialized = syncNode2Str(pSyncNode); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); initRaftId(pSyncNode); @@ -104,7 +104,7 @@ int main(int argc, char** argv) { char* serialized = voteGranted2Str(pVotesGranted); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } SyncTerm term = 1234; @@ -114,7 +114,7 @@ int main(int argc, char** argv) { char* serialized = voteGranted2Str(pVotesGranted); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } for (int i = 0; i < replicaNum; ++i) { @@ -129,7 +129,7 @@ int main(int argc, char** argv) { char* serialized = voteGranted2Str(pVotesGranted); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } voteGrantedVote(pVotesGranted, reply); @@ -137,7 +137,7 @@ int main(int argc, char** argv) { char* serialized = voteGranted2Str(pVotesGranted); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } } @@ -147,7 +147,7 @@ int main(int argc, char** argv) { char* serialized = voteGranted2Str(pVotesGranted); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } voteGrantedDestroy(pVotesGranted); diff --git a/source/libs/sync/test/syncVotesRespondTest.cpp b/source/libs/sync/test/syncVotesRespondTest.cpp index 76fd6fab4e1d807565d3f79f0d0910b6f8111953..39ba6253a1a61974623939ed055dc51247d7eab3 100644 --- a/source/libs/sync/test/syncVotesRespondTest.cpp +++ b/source/libs/sync/test/syncVotesRespondTest.cpp @@ -67,7 +67,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } @@ -92,7 +92,7 @@ int main(int argc, char** argv) { char* serialized = syncNode2Str(pSyncNode); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); initRaftId(pSyncNode); @@ -104,7 +104,7 @@ int main(int argc, char** argv) { char* serialized = votesRespond2Str(pVotesRespond); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } SyncTerm term = 1234; @@ -114,7 +114,7 @@ int main(int argc, char** argv) { char* serialized = votesRespond2Str(pVotesRespond); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } for (int i = 0; i < replicaNum; ++i) { @@ -129,7 +129,7 @@ int main(int argc, char** argv) { char* serialized = votesRespond2Str(pVotesRespond); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } votesRespondAdd(pVotesRespond, reply); @@ -137,7 +137,7 @@ int main(int argc, char** argv) { char* serialized = votesRespond2Str(pVotesRespond); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } } @@ -147,7 +147,7 @@ int main(int argc, char** argv) { char* serialized = votesRespond2Str(pVotesRespond); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } votesRespondDestory(pVotesRespond); diff --git a/source/libs/sync/test/syncWriteTest.cpp b/source/libs/sync/test/syncWriteTest.cpp index 2598abbddd86ac6813afd287246202d5439cb9c8..a3c24f69ff7d5981cd8ad19f97bb9e9fa7866eec 100644 --- a/source/libs/sync/test/syncWriteTest.cpp +++ b/source/libs/sync/test/syncWriteTest.cpp @@ -28,23 +28,33 @@ SSyncFSM * pFsm; SWal * pWal; SSyncNode *gSyncNode; -void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) { - printf("==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code); - syncRpcMsgPrint2((char *)"==CommitCb==", (SRpcMsg *)pBuf); +void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); } -void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) { - printf("==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code); - syncRpcMsgPrint2((char *)"==PreCommitCb==", (SRpcMsg *)pBuf); +void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, index, isWeak, + code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); } -void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pBuf, SyncIndex index, bool isWeak, int32_t code) { - printf("==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d \n", pFsm, index, isWeak, code); - syncRpcMsgPrint2((char *)"==RollBackCb==", (SRpcMsg *)pBuf); +void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); } void initFsm() { - pFsm = (SSyncFSM *)malloc(sizeof(SSyncFSM)); + pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM)); pFsm->FpCommitCb = CommitCb; pFsm->FpPreCommitCb = PreCommitCb; pFsm->FpRollBackCb = RollBackCb; @@ -108,16 +118,16 @@ void initRaftId(SSyncNode *pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char *s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } SRpcMsg *step0() { - SRpcMsg *pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + SRpcMsg *pMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); memset(pMsg, 0, sizeof(SRpcMsg)); pMsg->msgType = 9999; pMsg->contLen = 32; - pMsg->pCont = malloc(pMsg->contLen); + pMsg->pCont = taosMemoryMalloc(pMsg->contLen); snprintf((char *)(pMsg->pCont), pMsg->contLen, "hello, world"); return pMsg; } diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index c6e8c9dca9496f1e530c1caabeee3f35ef69c1ae..f9dec34d126f41830dc0b8b2059308ef80c2431d 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -73,7 +73,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, S *ppBt = NULL; - pBt = (SBTree *)calloc(1, sizeof(*pBt)); + pBt = (SBTree *)taosMemoryCalloc(1, sizeof(*pBt)); if (pBt == NULL) { return -1; } @@ -107,7 +107,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, S // TODO: pBt->root ret = tdbBtreeOpenImpl(pBt); if (ret < 0) { - free(pBt); + taosMemoryFree(pBt); return -1; } @@ -161,7 +161,7 @@ int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *p // TODO: refact code here pBt = pCur->pBt; if (!pBt->pTmp) { - pBt->pTmp = (u8 *)malloc(pBt->pageSize); + pBt->pTmp = (u8 *)taosMemoryMalloc(pBt->pageSize); if (pBt->pTmp == NULL) { return -1; } diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 00f38a19bbee7967d18069ed41ceb30baad32e10..8360c925a8ba9f2f4596bf1895f41fcc36eb27e1 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -30,7 +30,7 @@ int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprF *ppDb = NULL; - pDb = (STDb *)calloc(1, sizeof(*pDb)); + pDb = (STDb *)taosMemoryCalloc(1, sizeof(*pDb)); if (pDb == NULL) { return -1; } diff --git a/source/libs/tdb/src/db/tdbEnv.c b/source/libs/tdb/src/db/tdbEnv.c index 9a4dcdbcd59a72b10740e3754339bb312be745d0..fc9861ff8fd7c2853bfcfc3e5995125a74c2f846 100644 --- a/source/libs/tdb/src/db/tdbEnv.c +++ b/source/libs/tdb/src/db/tdbEnv.c @@ -27,7 +27,7 @@ int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, STEnv **ppEnv) dsize = strlen(rootDir); zsize = sizeof(*pEnv) + dsize * 2 + strlen(TDB_JOURNAL_NAME) + 3; - pPtr = (uint8_t *)calloc(1, zsize); + pPtr = (uint8_t *)taosMemoryCalloc(1, zsize); if (pPtr == NULL) { return -1; } diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 9d7181c1da292e9a9f5b242ebbc1235d3f2f0b00..c9708ababd98a69ff0578416abce2db653adce67 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -65,7 +65,7 @@ int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) { void *pPtr; SPage *pPgHdr; - pCache = (SPCache *)calloc(1, sizeof(*pCache)); + pCache = (SPCache *)taosMemoryCalloc(1, sizeof(*pCache)); if (pCache == NULL) { return -1; } @@ -74,7 +74,7 @@ int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) { pCache->cacheSize = cacheSize; if (tdbPCacheOpenImpl(pCache) < 0) { - free(pCache); + taosMemoryFree(pCache); return -1; } @@ -281,7 +281,7 @@ static int tdbPCacheOpenImpl(SPCache *pCache) { // Open the hash table pCache->nPage = 0; pCache->nHash = pCache->cacheSize; - pCache->pgHash = (SPage **)calloc(pCache->nHash, sizeof(SPage *)); + pCache->pgHash = (SPage **)taosMemoryCalloc(pCache->nHash, sizeof(SPage *)); if (pCache->pgHash == NULL) { // TODO return -1; @@ -301,9 +301,9 @@ int tdbPCacheGetPageSize(SPCache *pCache) { return pCache->pageSize; } static void *tdbOsMalloc(void *arg, size_t size) { void *ptr; - ptr = malloc(size); + ptr = taosMemoryMalloc(size); return ptr; } -static void tdbOsFree(void *arg, void *ptr) { free(ptr); } \ No newline at end of file +static void tdbOsFree(void *arg, void *ptr) { taosMemoryFree(ptr); } \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index ac0bc15e1d5ce768aa5a51931cef121765f4ae30..a181c2792663e0cb0065356f9b9bafa09207bfc1 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -60,7 +60,7 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { zsize = sizeof(*pPager) /* SPager */ + fsize + 1 /* dbFileName */ + fsize + 8 + 1; /* jFileName */ - pPtr = (uint8_t *)calloc(1, zsize); + pPtr = (uint8_t *)taosMemoryCalloc(1, zsize); if (pPtr == NULL) { return -1; } diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index bfe7904a63e3c64b9c1e0c9a19c8ff784c9998c0..e50045d30b41abb363d28f5958645829d2c893ee 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -30,7 +30,7 @@ STfs *tfsOpen(SDiskCfg *pCfg, int32_t ndisk) { return NULL; } - STfs *pTfs = calloc(1, sizeof(STfs)); + STfs *pTfs = taosMemoryCalloc(1, sizeof(STfs)); if (pTfs == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -86,7 +86,7 @@ void tfsClose(STfs *pTfs) { taosHashCleanup(pTfs->hash); taosThreadSpinDestroy(&pTfs->lock); - free(pTfs); + taosMemoryFree(pTfs); } void tfsUpdateSize(STfs *pTfs) { @@ -184,7 +184,7 @@ void *tfsDecodeFile(STfs *pTfs, void *buf, STfsFile *pFile) { tfsInitFile(pTfs, pFile, diskId, rname); - tfree(rname); + taosMemoryFreeClear(rname); return buf; } @@ -242,12 +242,12 @@ int32_t tfsMkdirRecurAt(STfs *pTfs, const char *rname, SDiskID diskId) { char *dir = strdup(taosDirName(s)); if (tfsMkdirRecurAt(pTfs, dir, diskId) < 0) { - free(s); - free(dir); + taosMemoryFree(s); + taosMemoryFree(dir); return -1; } - free(s); - free(dir); + taosMemoryFree(s); + taosMemoryFree(dir); if (tfsMkdirAt(pTfs, rname, diskId) < 0) { return -1; @@ -311,7 +311,7 @@ int32_t tfsRename(STfs *pTfs, char *orname, char *nrname) { } STfsDir *tfsOpendir(STfs *pTfs, const char *rname) { - STfsDir *pDir = calloc(1, sizeof(STfsDir)); + STfsDir *pDir = taosMemoryCalloc(1, sizeof(STfsDir)); if (pDir == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -323,7 +323,7 @@ STfsDir *tfsOpendir(STfs *pTfs, const char *rname) { tstrncpy(pDir->dirname, rname, TSDB_FILENAME_LEN); if (tfsOpendirImpl(pTfs, pDir) < 0) { - free(pDir); + taosMemoryFree(pDir); return NULL; } @@ -369,7 +369,7 @@ void tfsClosedir(STfsDir *pTfsDir) { taosCloseDir(pTfsDir->pDir); pTfsDir->pDir = NULL; } - free(pTfsDir); + taosMemoryFree(pTfsDir); } } diff --git a/source/libs/tfs/src/tfsDisk.c b/source/libs/tfs/src/tfsDisk.c index 52396db3be470bcb9735f10bef8789ec883089a3..ff40529ab2d75ec90be9b35ad259594f869d048e 100644 --- a/source/libs/tfs/src/tfsDisk.c +++ b/source/libs/tfs/src/tfsDisk.c @@ -17,7 +17,7 @@ #include "tfsInt.h" STfsDisk *tfsNewDisk(int32_t level, int32_t id, const char *path) { - STfsDisk *pDisk = calloc(1, sizeof(STfsDisk)); + STfsDisk *pDisk = taosMemoryCalloc(1, sizeof(STfsDisk)); if (pDisk == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -25,7 +25,7 @@ STfsDisk *tfsNewDisk(int32_t level, int32_t id, const char *path) { pDisk->path = strdup(path); if (pDisk->path == NULL) { - free(pDisk); + taosMemoryFree(pDisk); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -38,8 +38,8 @@ STfsDisk *tfsNewDisk(int32_t level, int32_t id, const char *path) { STfsDisk *tfsFreeDisk(STfsDisk *pDisk) { if (pDisk != NULL) { - free(pDisk->path); - free(pDisk); + taosMemoryFree(pDisk->path); + taosMemoryFree(pDisk); } return NULL; diff --git a/source/libs/tfs/test/tfsTest.cpp b/source/libs/tfs/test/tfsTest.cpp index 9cb914a670e58ad6ca6f541a9677ed32dfd55925..1a093c3877a3c47e73af34cfc729887062d98e3a 100644 --- a/source/libs/tfs/test/tfsTest.cpp +++ b/source/libs/tfs/test/tfsTest.cpp @@ -196,7 +196,7 @@ TEST_F(TfsTest, 04_File) { { int32_t size = 1024; - void *ret = malloc(size + sizeof(size_t)); + void *ret = taosMemoryMalloc(size + sizeof(size_t)); *(size_t *)ret = size; void *buf = (void *)((char *)ret + sizeof(size_t)); diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 8cfde8267dd58472ee687dd051b1724325e8398e..eaca66aeab70ef9c9978f3671c1f17a826746f31 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -281,14 +281,14 @@ void* transCtxDumpVal(STransCtx* ctx, int32_t key); // queue sending msgs typedef struct { SArray* q; - void (*free)(void* arg); + void (*freeFunc)(const void* arg); } STransQueue; /* * init queue * note: queue'size is small, default 1 */ -void transQueueInit(STransQueue* queue, void (*free)(void* arg)); +void transQueueInit(STransQueue* queue, void (*freeFunc)(const void* arg)); /* * put arg into queue diff --git a/source/libs/transport/src/rpcCache.c b/source/libs/transport/src/rpcCache.c index d629f6dba4b958137756bbb867df4e9c23018aa4..e02de2d8b96f628d8eb07355690b968840378863 100644 --- a/source/libs/transport/src/rpcCache.c +++ b/source/libs/transport/src/rpcCache.c @@ -60,21 +60,21 @@ void *rpcOpenConnCache(int maxSessions, void (*cleanFp)(void *), void *tmrCtrl, connHashMemPool = taosMemPoolInit(maxSessions, sizeof(SConnHash)); if (connHashMemPool == 0) return NULL; - connHashList = calloc(sizeof(SConnHash *), maxSessions); + connHashList = taosMemoryCalloc(sizeof(SConnHash *), maxSessions); if (connHashList == 0) { taosMemPoolCleanUp(connHashMemPool); return NULL; } - pCache = malloc(sizeof(SConnCache)); + pCache = taosMemoryMalloc(sizeof(SConnCache)); if (pCache == NULL) { taosMemPoolCleanUp(connHashMemPool); - free(connHashList); + taosMemoryFree(connHashList); return NULL; } memset(pCache, 0, sizeof(SConnCache)); - pCache->count = calloc(sizeof(int), maxSessions); + pCache->count = taosMemoryCalloc(sizeof(int), maxSessions); pCache->total = 0; pCache->keepTimer = keepTimer; pCache->maxSessions = maxSessions; @@ -82,7 +82,7 @@ void *rpcOpenConnCache(int maxSessions, void (*cleanFp)(void *), void *tmrCtrl, pCache->connHashList = connHashList; pCache->cleanFp = cleanFp; pCache->tmrCtrl = tmrCtrl; - pCache->lockedBy = calloc(sizeof(int64_t), maxSessions); + pCache->lockedBy = taosMemoryCalloc(sizeof(int64_t), maxSessions); taosTmrReset(rpcCleanConnCache, (int32_t)(pCache->keepTimer * 2), pCache, pCache->tmrCtrl, &pCache->pTimer); taosThreadMutexInit(&pCache->mutex, NULL); @@ -102,16 +102,16 @@ void rpcCloseConnCache(void *handle) { if (pCache->connHashMemPool) taosMemPoolCleanUp(pCache->connHashMemPool); - tfree(pCache->connHashList); - tfree(pCache->count); - tfree(pCache->lockedBy); + taosMemoryFreeClear(pCache->connHashList); + taosMemoryFreeClear(pCache->count); + taosMemoryFreeClear(pCache->lockedBy); taosThreadMutexUnlock(&pCache->mutex); taosThreadMutexDestroy(&pCache->mutex); memset(pCache, 0, sizeof(SConnCache)); - free(pCache); + taosMemoryFree(pCache); } void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, int8_t connType) { diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 98b24ac3c7cb4e6404b94b8780c2fdbcb2d16f2f..6beda5bfed11c3a42bf03457b000bf295792d673 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -208,7 +208,7 @@ static void rpcDecRef(SRpcInfo *pRpc); static void rpcFree(void *p) { tTrace("free mem: %p", p); - free(p); + taosMemoryFree(p); } static void rpcInitImp(void) { @@ -240,7 +240,7 @@ void *rpcOpen(const SRpcInit *pInit) { // taosThreadOnce(&tsRpcInit, rpcInit); - pRpc = (SRpcInfo *)calloc(1, sizeof(SRpcInfo)); + pRpc = (SRpcInfo *)taosMemoryCalloc(1, sizeof(SRpcInfo)); if (pRpc == NULL) return NULL; if (pInit->label) tstrncpy(pRpc->label, pInit->label, tListLen(pInit->label)); @@ -270,7 +270,7 @@ void *rpcOpen(const SRpcInit *pInit) { atomic_add_fetch_32(&tsRpcNum, 1); size_t size = sizeof(SRpcConn) * pRpc->sessions; - pRpc->connList = (SRpcConn *)calloc(1, size); + pRpc->connList = (SRpcConn *)taosMemoryCalloc(1, size); if (pRpc->connList == NULL) { tError("%s failed to allocate memory for taos connections, size:%" PRId64, pRpc->label, (int64_t)size); rpcClose(pRpc); @@ -350,7 +350,7 @@ void rpcClose(void *param) { void *rpcMallocCont(int contLen) { int size = contLen + RPC_MSG_OVERHEAD; - char *start = (char *)calloc(1, (size_t)size); + char *start = (char *)taosMemoryCalloc(1, (size_t)size); if (start == NULL) { tError("failed to malloc msg, size:%d", size); return NULL; @@ -364,7 +364,7 @@ void *rpcMallocCont(int contLen) { void rpcFreeCont(void *cont) { if (cont) { char *temp = ((char *)cont) - sizeof(SRpcHead) - sizeof(SRpcReqContext); - free(temp); + taosMemoryFree(temp); tTrace("free mem: %p", temp); } } @@ -374,12 +374,12 @@ void *rpcReallocCont(void *ptr, int contLen) { char *start = ((char *)ptr) - sizeof(SRpcReqContext) - sizeof(SRpcHead); if (contLen == 0) { - free(start); + taosMemoryFree(start); return NULL; } int size = contLen + RPC_MSG_OVERHEAD; - start = realloc(start, size); + start = taosMemoryRealloc(start, size); if (start == NULL) { tError("failed to realloc cont, size:%d", size); return NULL; @@ -574,7 +574,7 @@ void rpcCancelRequest(int64_t rid) { static void rpcFreeMsg(void *msg) { if (msg) { char *temp = (char *)msg - sizeof(SRpcReqContext); - free(temp); + taosMemoryFree(temp); tTrace("free mem: %p", temp); } } @@ -1039,7 +1039,7 @@ static void doRpcReportBrokenLinkToServer(void *param, void *id) { SRpcConn *pConn = (SRpcConn *)(pRpcMsg->handle); SRpcInfo *pRpc = pConn->pRpc; (*(pRpc->cfp))(pRpc->parent, pRpcMsg, NULL); - free(pRpcMsg); + taosMemoryFree(pRpcMsg); } static void rpcReportBrokenLinkToServer(SRpcConn *pConn) { SRpcInfo *pRpc = pConn->pRpc; @@ -1049,7 +1049,7 @@ static void rpcReportBrokenLinkToServer(SRpcConn *pConn) { rpcAddRef(pRpc); tDebug("%s, notify the server app, connection is gone", pConn->info); - SRpcMsg *rpcMsg = malloc(sizeof(SRpcMsg)); + SRpcMsg *rpcMsg = taosMemoryMalloc(sizeof(SRpcMsg)); rpcMsg->pCont = pConn->pReqMsg; // pReqMsg is re-used to store the APP context from server rpcMsg->contLen = pConn->reqMsgLen; // reqMsgLen is re-used to store the APP context length rpcMsg->ahandle = pConn->ahandle; @@ -1061,7 +1061,7 @@ static void rpcReportBrokenLinkToServer(SRpcConn *pConn) { if (pRpc->cfp) { taosTmrStart(doRpcReportBrokenLinkToServer, 0, rpcMsg, pRpc->tmrCtrl); } else { - free(rpcMsg); + taosMemoryFree(rpcMsg); } } @@ -1484,7 +1484,7 @@ static int32_t rpcCompressRpcMsg(char *pCont, int32_t contLen) { return contLen; } - char *buf = malloc(contLen + overhead + 8); // 8 extra bytes + char *buf = taosMemoryMalloc(contLen + overhead + 8); // 8 extra bytes if (buf == NULL) { tError("failed to allocate memory for rpc msg compression, contLen:%d", contLen); return contLen; @@ -1510,7 +1510,7 @@ static int32_t rpcCompressRpcMsg(char *pCont, int32_t contLen) { finalLen = contLen; } - free(buf); + taosMemoryFree(buf); return finalLen; } @@ -1526,7 +1526,7 @@ static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead) { int contLen = htonl(pComp->contLen); // prepare the temporary buffer to decompress message - char *temp = (char *)malloc(contLen + RPC_MSG_OVERHEAD); + char *temp = (char *)taosMemoryMalloc(contLen + RPC_MSG_OVERHEAD); pNewHead = (SRpcHead *)(temp + sizeof(SRpcReqContext)); // reserve SRpcReqContext if (pNewHead) { @@ -1671,10 +1671,10 @@ static void rpcDecRef(SRpcInfo *pRpc) { taosTmrCleanUp(pRpc->tmrCtrl); taosIdPoolCleanUp(pRpc->idPool); - tfree(pRpc->connList); + taosMemoryFreeClear(pRpc->connList); taosThreadMutexDestroy(&pRpc->mutex); tDebug("%s rpc resources are released", pRpc->label); - tfree(pRpc); + taosMemoryFreeClear(pRpc); atomic_sub_fetch_32(&tsRpcNum, 1); } diff --git a/source/libs/transport/src/rpcTcp.c b/source/libs/transport/src/rpcTcp.c index 4b1700a10f882c64fa4f74cda67634b782ba3421..52c5ddcf631e9a0cc3a307fab00122c5139fa5dd 100644 --- a/source/libs/transport/src/rpcTcp.c +++ b/source/libs/transport/src/rpcTcp.c @@ -78,7 +78,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread SServerObj *pServerObj; SThreadObj *pThreadObj; - pServerObj = (SServerObj *)calloc(sizeof(SServerObj), 1); + pServerObj = (SServerObj *)taosMemoryCalloc(sizeof(SServerObj), 1); if (pServerObj == NULL) { tError("TCP:%s no enough memory", label); terrno = TAOS_SYSTEM_ERROR(errno); @@ -92,11 +92,11 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread tstrncpy(pServerObj->label, label, sizeof(pServerObj->label)); pServerObj->numOfThreads = numOfThreads; - pServerObj->pThreadObj = (SThreadObj **)calloc(sizeof(SThreadObj *), numOfThreads); + pServerObj->pThreadObj = (SThreadObj **)taosMemoryCalloc(sizeof(SThreadObj *), numOfThreads); if (pServerObj->pThreadObj == NULL) { tError("TCP:%s no enough memory", label); terrno = TAOS_SYSTEM_ERROR(errno); - free(pServerObj); + taosMemoryFree(pServerObj); return NULL; } @@ -107,13 +107,13 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread // initialize parameters in case it may encounter error later for (int i = 0; i < numOfThreads; ++i) { - pThreadObj = (SThreadObj *)calloc(sizeof(SThreadObj), 1); + pThreadObj = (SThreadObj *)taosMemoryCalloc(sizeof(SThreadObj), 1); if (pThreadObj == NULL) { tError("TCP:%s no enough memory", label); terrno = TAOS_SYSTEM_ERROR(errno); - for (int j = 0; j < i; ++j) free(pServerObj->pThreadObj[j]); - free(pServerObj->pThreadObj); - free(pServerObj); + for (int j = 0; j < i; ++j) taosMemoryFree(pServerObj->pThreadObj[j]); + taosMemoryFree(pServerObj->pThreadObj); + taosMemoryFree(pServerObj); return NULL; } @@ -222,8 +222,8 @@ void taosCleanUpTcpServer(void *handle) { tDebug("%s TCP server is cleaned up", pServerObj->label); - tfree(pServerObj->pThreadObj); - tfree(pServerObj); + taosMemoryFreeClear(pServerObj->pThreadObj); + taosMemoryFreeClear(pServerObj); } static void *taosAcceptTcpConnection(void *arg) { @@ -290,7 +290,7 @@ static void *taosAcceptTcpConnection(void *arg) { } void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) { - SClientObj *pClientObj = (SClientObj *)calloc(1, sizeof(SClientObj)); + SClientObj *pClientObj = (SClientObj *)taosMemoryCalloc(1, sizeof(SClientObj)); if (pClientObj == NULL) { tError("TCP:%s no enough memory", label); terrno = TAOS_SYSTEM_ERROR(errno); @@ -299,10 +299,10 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread tstrncpy(pClientObj->label, label, sizeof(pClientObj->label)); pClientObj->numOfThreads = numOfThreads; - pClientObj->pThreadObj = (SThreadObj **)calloc(numOfThreads, sizeof(SThreadObj *)); + pClientObj->pThreadObj = (SThreadObj **)taosMemoryCalloc(numOfThreads, sizeof(SThreadObj *)); if (pClientObj->pThreadObj == NULL) { tError("TCP:%s no enough memory", label); - tfree(pClientObj); + taosMemoryFreeClear(pClientObj); terrno = TAOS_SYSTEM_ERROR(errno); } @@ -312,12 +312,12 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); for (int i = 0; i < numOfThreads; ++i) { - SThreadObj *pThreadObj = (SThreadObj *)calloc(1, sizeof(SThreadObj)); + SThreadObj *pThreadObj = (SThreadObj *)taosMemoryCalloc(1, sizeof(SThreadObj)); if (pThreadObj == NULL) { tError("TCP:%s no enough memory", label); terrno = TAOS_SYSTEM_ERROR(errno); - for (int j = 0; j < i; ++j) free(pClientObj->pThreadObj[j]); - free(pClientObj); + for (int j = 0; j < i; ++j) taosMemoryFree(pClientObj->pThreadObj[j]); + taosMemoryFree(pClientObj); taosThreadAttrDestroy(&thattr); return NULL; } @@ -378,8 +378,8 @@ void taosCleanUpTcpClient(void *chandle) { } tDebug("%s TCP client is cleaned up", pClientObj->label); - tfree(pClientObj->pThreadObj); - tfree(pClientObj); + taosMemoryFreeClear(pClientObj->pThreadObj); + taosMemoryFreeClear(pClientObj); } void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { @@ -477,9 +477,9 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); int32_t size = msgLen + tsRpcOverhead; - buffer = malloc(size); + buffer = taosMemoryMalloc(size); if (NULL == buffer) { - tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); + tError("%s %p TCP taosMemoryMalloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); return -1; } else { tTrace("%s %p read data, FD:%p TCP malloc mem:%p", pThreadObj->label, pFdObj->thandle, pFdObj, buffer); @@ -491,7 +491,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { if (leftLen != retLen) { tError("%s %p read error, leftLen:%d retLen:%d FD:%p", pThreadObj->label, pFdObj->thandle, leftLen, retLen, pFdObj); - free(buffer); + taosMemoryFree(buffer); return -1; } @@ -507,7 +507,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { pInfo->connType = RPC_CONN_TCP; if (pFdObj->closedByApp) { - free(buffer); + taosMemoryFree(buffer); return -1; } @@ -580,7 +580,7 @@ static void *taosProcessTcpData(void *param) { taosThreadMutexDestroy(&(pThreadObj->mutex)); tDebug("%s TCP thread exits ...", pThreadObj->label); - tfree(pThreadObj); + taosMemoryFreeClear(pThreadObj); return NULL; } @@ -588,7 +588,7 @@ static void *taosProcessTcpData(void *param) { static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, TdSocketPtr pSocket) { struct epoll_event event; - SFdObj *pFdObj = (SFdObj *)calloc(sizeof(SFdObj), 1); + SFdObj *pFdObj = (SFdObj *)taosMemoryCalloc(sizeof(SFdObj), 1); if (pFdObj == NULL) { return NULL; } @@ -601,7 +601,7 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, TdSocketPtr pSocket) { event.events = EPOLLIN | EPOLLRDHUP; event.data.ptr = pFdObj; if (taosCtlEpoll(pThreadObj->pEpoll, EPOLL_CTL_ADD, pSocket, &event) < 0) { - tfree(pFdObj); + taosMemoryFreeClear(pFdObj); terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } @@ -652,6 +652,6 @@ static void taosFreeFdObj(SFdObj *pFdObj) { tDebug("%s %p TCP connection is closed, FD:%p numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds); - tfree(pFdObj); + taosMemoryFreeClear(pFdObj); } #endif \ No newline at end of file diff --git a/source/libs/transport/src/rpcUdp.c b/source/libs/transport/src/rpcUdp.c index af26761603f73082ad7dabffb6d7961aee6f3c0c..359a94011dc493493aa9c63660806787b3be4d11 100644 --- a/source/libs/transport/src/rpcUdp.c +++ b/source/libs/transport/src/rpcUdp.c @@ -62,7 +62,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads SUdpConnSet *pSet; int size = (int)sizeof(SUdpConnSet) + threads * (int)sizeof(SUdpConn); - pSet = (SUdpConnSet *)malloc((size_t)size); + pSet = (SUdpConnSet *)taosMemoryMalloc((size_t)size); if (pSet == NULL) { tError("%s failed to allocate UdpConn", label); terrno = TAOS_SYSTEM_ERROR(errno); @@ -92,7 +92,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads break; } - pConn->buffer = malloc(RPC_MAX_UDP_SIZE); + pConn->buffer = taosMemoryMalloc(RPC_MAX_UDP_SIZE); if (NULL == pConn->buffer) { tError("%s failed to malloc recv buffer", label); break; @@ -148,7 +148,7 @@ void taosStopUdpConnection(void *handle) { if (taosCheckPthreadValid(pConn->thread)) { taosThreadJoin(pConn->thread, NULL); } - tfree(pConn->buffer); + taosMemoryFreeClear(pConn->buffer); // tTrace("%s UDP thread is closed, index:%d", pConn->label, i); } @@ -167,7 +167,7 @@ void taosCleanUpUdpConnection(void *handle) { } tDebug("%s UDP is cleaned up", pSet->label); - tfree(pSet); + taosMemoryFreeClear(pSet); } void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { @@ -219,7 +219,7 @@ static void *taosRecvUdpData(void *param) { } int32_t size = dataLen + tsRpcOverhead; - char * tmsg = malloc(size); + char * tmsg = taosMemoryMalloc(size); if (NULL == tmsg) { tError("%s failed to allocate memory, size:%" PRId64, pConn->label, (int64_t)dataLen); continue; diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 4e2cb835279b7b9ae951b8094704bd5c98577fbd..90f15dd7d052b637f473e56a0dfb88c9963a987b 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -47,7 +47,7 @@ static int32_t taosBuildHttpHeader(const char* server, int32_t contLen, char* pH int32_t taosCompressHttpRport(char* pSrc, int32_t srcLen) { int32_t code = -1; int32_t destLen = srcLen; - void* pDest = malloc(destLen); + void* pDest = taosMemoryMalloc(destLen); if (pDest == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -109,7 +109,7 @@ _OVER: code = gzipStream.total_out; } - free(pDest); + taosMemoryFree(pDest); return code; } @@ -145,7 +145,7 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 uv_tcp_t socket_tcp = {0}; uv_loop_t* loop = uv_default_loop(); uv_tcp_init(loop, &socket_tcp); - uv_connect_t* connect = (uv_connect_t*)malloc(sizeof(uv_connect_t)); + uv_connect_t* connect = (uv_connect_t*)taosMemoryMalloc(sizeof(uv_connect_t)); if (flag == HTTP_GZIP) { int32_t dstLen = taosCompressHttpRport(pCont, contLen); @@ -168,7 +168,7 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 terrno = 0; uv_run(loop, UV_RUN_DEFAULT); uv_loop_close(loop); - free(connect); + taosMemoryFree(connect); return terrno; } diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 9d0fba488510a5c5492fda3795b007760b5d0936..ebb90338cd42303243f44be81bceef25f1160b63 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -28,7 +28,7 @@ void (*taosUnRefHandle[])(void* handle) = {transUnrefSrvHandle, transUnrefCliHan void (*transReleaseHandle[])(void* handle) = {transReleaseSrvHandle, transReleaseCliHandle}; void* rpcOpen(const SRpcInit* pInit) { - SRpcInfo* pRpc = calloc(1, sizeof(SRpcInfo)); + SRpcInfo* pRpc = taosMemoryCalloc(1, sizeof(SRpcInfo)); if (pRpc == NULL) { return NULL; } @@ -61,13 +61,13 @@ void* rpcOpen(const SRpcInit* pInit) { void rpcClose(void* arg) { SRpcInfo* pRpc = (SRpcInfo*)arg; (*taosCloseHandle[pRpc->connType])(pRpc->tcphandle); - free(pRpc); + taosMemoryFree(pRpc); return; } void* rpcMallocCont(int contLen) { int size = contLen + TRANS_MSG_OVERHEAD; - char* start = (char*)calloc(1, (size_t)size); + char* start = (char*)taosMemoryCalloc(1, (size_t)size); if (start == NULL) { tError("failed to malloc msg, size:%d", size); return NULL; @@ -81,7 +81,7 @@ void rpcFreeCont(void* cont) { if (cont == NULL) { return; } - free((char*)cont - TRANS_MSG_OVERHEAD); + taosMemoryFree((char*)cont - TRANS_MSG_OVERHEAD); } void* rpcReallocCont(void* ptr, int contLen) { if (ptr == NULL) { @@ -89,7 +89,7 @@ void* rpcReallocCont(void* ptr, int contLen) { } char* st = (char*)ptr - TRANS_MSG_OVERHEAD; int sz = contLen + TRANS_MSG_OVERHEAD; - st = realloc(st, sz); + st = taosMemoryRealloc(st, sz); if (st == NULL) { return NULL; } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index c0ee9b9ca5bf8e0a8206810af27118c0f94e52f7..a0e8af1db96d9f0385752011be10b6cd861a7af0 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -429,9 +429,9 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { } static SCliConn* cliCreateConn(SCliThrdObj* pThrd) { - SCliConn* conn = calloc(1, sizeof(SCliConn)); + SCliConn* conn = taosMemoryCalloc(1, sizeof(SCliConn)); // read/write stream handle - conn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); + conn->stream = (uv_stream_t*)taosMemoryMalloc(sizeof(uv_tcp_t)); uv_tcp_init(pThrd->loop, (uv_tcp_t*)(conn->stream)); conn->stream->data = conn; @@ -456,12 +456,12 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { } static void cliDestroy(uv_handle_t* handle) { SCliConn* conn = handle->data; - free(conn->ip); - free(conn->stream); + taosMemoryFree(conn->ip); + taosMemoryFree(conn->stream); transCtxCleanup(&conn->ctx); transQueueDestroy(&conn->cliMsgs); tTrace("%s cli conn %p destroy successfully", CONN_GET_INST_LABEL(conn), conn); - free(conn); + taosMemoryFree(conn); } static bool cliHandleNoResp(SCliConn* conn) { bool res = false; @@ -519,7 +519,7 @@ void cliSend(SCliConn* pConn) { int msgLen = transMsgLenFromCont(pMsg->contLen); if (!pConn->secured) { - char* buf = calloc(1, msgLen + sizeof(STransUserMsg)); + char* buf = taosMemoryCalloc(1, msgLen + sizeof(STransUserMsg)); memcpy(buf, (char*)pHead, msgLen); STransUserMsg* uMsg = (STransUserMsg*)(buf + msgLen); @@ -697,12 +697,12 @@ static void* cliWorkThread(void* arg) { } void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { - SCliObj* cli = calloc(1, sizeof(SCliObj)); + SCliObj* cli = taosMemoryCalloc(1, sizeof(SCliObj)); STrans* pTransInst = shandle; memcpy(cli->label, label, strlen(label)); cli->numOfThreads = numOfThreads; - cli->pThreadObj = (SCliThrdObj**)calloc(cli->numOfThreads, sizeof(SCliThrdObj*)); + cli->pThreadObj = (SCliThrdObj**)taosMemoryCalloc(cli->numOfThreads, sizeof(SCliThrdObj*)); for (int i = 0; i < cli->numOfThreads; i++) { SCliThrdObj* pThrd = createThrdObj(); @@ -731,16 +731,16 @@ static void destroyCmsg(SCliMsg* pMsg) { } transDestroyConnCtx(pMsg->ctx); destroyUserdata(&pMsg->msg); - free(pMsg); + taosMemoryFree(pMsg); } static SCliThrdObj* createThrdObj() { - SCliThrdObj* pThrd = (SCliThrdObj*)calloc(1, sizeof(SCliThrdObj)); + SCliThrdObj* pThrd = (SCliThrdObj*)taosMemoryCalloc(1, sizeof(SCliThrdObj)); QUEUE_INIT(&pThrd->msg); taosThreadMutexInit(&pThrd->msgMtx, NULL); - pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); + pThrd->loop = (uv_loop_t*)taosMemoryMalloc(sizeof(uv_loop_t)); uv_loop_init(pThrd->loop); pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, 5, pThrd, cliAsyncCb); @@ -763,20 +763,20 @@ static void destroyThrdObj(SCliThrdObj* pThrd) { transDestroyAsyncPool(pThrd->asyncPool); uv_timer_stop(&pThrd->timer); - free(pThrd->loop); - free(pThrd); + taosMemoryFree(pThrd->loop); + taosMemoryFree(pThrd); } static void transDestroyConnCtx(STransConnCtx* ctx) { if (ctx != NULL) { - free(ctx->ip); + taosMemoryFree(ctx->ip); } - free(ctx); + taosMemoryFree(ctx); } // void cliSendQuit(SCliThrdObj* thrd) { // cli can stop gracefully - SCliMsg* msg = calloc(1, sizeof(SCliMsg)); + SCliMsg* msg = taosMemoryCalloc(1, sizeof(SCliMsg)); msg->type = Quit; transSendAsync(thrd->asyncPool, &msg->q); } @@ -795,8 +795,8 @@ void transCloseClient(void* arg) { cliSendQuit(cli->pThreadObj[i]); destroyThrdObj(cli->pThreadObj[i]); } - free(cli->pThreadObj); - free(cli); + taosMemoryFree(cli->pThreadObj); + taosMemoryFree(cli); } void transRefCliHandle(void* handle) { if (handle == NULL) { @@ -822,7 +822,7 @@ void transReleaseCliHandle(void* handle) { } STransMsg tmsg = {.handle = handle}; - SCliMsg* cmsg = calloc(1, sizeof(SCliMsg)); + SCliMsg* cmsg = taosMemoryCalloc(1, sizeof(SCliMsg)); cmsg->msg = tmsg; cmsg->type = Release; @@ -836,7 +836,7 @@ void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* p index = cliRBChoseIdx(pTransInst); } - STransConnCtx* pCtx = calloc(1, sizeof(STransConnCtx)); + STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); pCtx->ahandle = pMsg->ahandle; pCtx->msgType = pMsg->msgType; pCtx->ip = strdup(ip); @@ -848,7 +848,7 @@ void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* p } assert(pTransInst->connType == TAOS_CONN_CLIENT); - SCliMsg* cliMsg = calloc(1, sizeof(SCliMsg)); + SCliMsg* cliMsg = taosMemoryCalloc(1, sizeof(SCliMsg)); cliMsg->ctx = pCtx; cliMsg->msg = *pMsg; cliMsg->st = taosGetTimestampUs(); @@ -867,17 +867,17 @@ void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pReq index = cliRBChoseIdx(pTransInst); } - STransConnCtx* pCtx = calloc(1, sizeof(STransConnCtx)); + STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); pCtx->ahandle = pReq->ahandle; pCtx->msgType = pReq->msgType; pCtx->ip = strdup(ip); pCtx->port = port; pCtx->hThrdIdx = index; - pCtx->pSem = calloc(1, sizeof(tsem_t)); + pCtx->pSem = taosMemoryCalloc(1, sizeof(tsem_t)); pCtx->pRsp = pRsp; tsem_init(pCtx->pSem, 0, 0); - SCliMsg* cliMsg = calloc(1, sizeof(SCliMsg)); + SCliMsg* cliMsg = taosMemoryCalloc(1, sizeof(SCliMsg)); cliMsg->ctx = pCtx; cliMsg->msg = *pReq; cliMsg->st = taosGetTimestampUs(); @@ -888,7 +888,7 @@ void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pReq tsem_t* pSem = pCtx->pSem; tsem_wait(pSem); tsem_destroy(pSem); - free(pSem); + taosMemoryFree(pSem); } #endif diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 209475ca051582e9a289a98e010666a8233a38da..31ac1e1fbf8cb84d89cd824757cb2874287a1cb3 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -52,7 +52,7 @@ bool transCompressMsg(char* msg, int32_t len, int32_t* flen) { return succ; } - char* buf = malloc(len + overhead + 8); // 8 extra bytes + char* buf = taosMemoryMalloc(len + overhead + 8); // 8 extra bytes if (buf == NULL) { tError("failed to allocate memory for rpc msg compression, contLen:%d", len); *flen = len; @@ -78,7 +78,7 @@ bool transCompressMsg(char* msg, int32_t len, int32_t* flen) { *flen = len; succ = false; } - free(buf); + taosMemoryFree(buf); return succ; } bool transDecompressMsg(char* msg, int32_t len, int32_t* flen) { @@ -92,15 +92,15 @@ bool transDecompressMsg(char* msg, int32_t len, int32_t* flen) { } void transConnCtxDestroy(STransConnCtx* ctx) { - free(ctx->ip); - free(ctx); + taosMemoryFree(ctx->ip); + taosMemoryFree(ctx); } void transFreeMsg(void* msg) { if (msg == NULL) { return; } - free((char*)msg - sizeof(STransMsgHead)); + taosMemoryFree((char*)msg - sizeof(STransMsgHead)); } int transInitBuffer(SConnBuffer* buf) { @@ -123,7 +123,7 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { SConnBuffer* p = connBuf; if (p->cap == 0) { - p->buf = (char*)calloc(CAPACITY, sizeof(char)); + p->buf = (char*)taosMemoryCalloc(CAPACITY, sizeof(char)); p->len = 0; p->cap = CAPACITY; p->total = -1; @@ -135,7 +135,7 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { uvBuf->len = CAPACITY - p->len; } else { p->cap = p->total; - p->buf = realloc(p->buf, p->cap); + p->buf = taosMemoryRealloc(p->buf, p->cap); uvBuf->base = p->buf + p->len; uvBuf->len = p->cap - p->len; @@ -160,7 +160,7 @@ int transPackMsg(STransMsgHead* msgHead, bool sercured, bool auth) { return 0; } int transUnpackMsg(STransMsgHead* msgHead) { return 0; } int transDestroyBuffer(SConnBuffer* buf) { if (buf->cap > 0) { - tfree(buf->buf); + taosMemoryFreeClear(buf->buf); } transClearBuffer(buf); @@ -174,16 +174,16 @@ int transSetConnOption(uv_tcp_t* stream) { } SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb) { - SAsyncPool* pool = calloc(1, sizeof(SAsyncPool)); + SAsyncPool* pool = taosMemoryCalloc(1, sizeof(SAsyncPool)); pool->index = 0; pool->nAsync = sz; - pool->asyncs = calloc(1, sizeof(uv_async_t) * pool->nAsync); + pool->asyncs = taosMemoryCalloc(1, sizeof(uv_async_t) * pool->nAsync); for (int i = 0; i < pool->nAsync; i++) { uv_async_t* async = &(pool->asyncs[i]); uv_async_init(loop, async, cb); - SAsyncItem* item = calloc(1, sizeof(SAsyncItem)); + SAsyncItem* item = taosMemoryCalloc(1, sizeof(SAsyncItem)); item->pThrd = arg; QUEUE_INIT(&item->qmsg); taosThreadMutexInit(&item->mtx, NULL); @@ -198,10 +198,10 @@ void transDestroyAsyncPool(SAsyncPool* pool) { SAsyncItem* item = async->data; taosThreadMutexDestroy(&item->mtx); - free(item); + taosMemoryFree(item); } - free(pool->asyncs); - free(pool); + taosMemoryFree(pool->asyncs); + taosMemoryFree(pool); } int transSendAsync(SAsyncPool* pool, queue* q) { int idx = pool->index; @@ -235,7 +235,7 @@ void transCtxCleanup(STransCtx* ctx) { STransCtxVal* iter = taosHashIterate(ctx->args, NULL); while (iter) { - iter->free(iter->val); + iter->freeFunc(iter->val); iter = taosHashIterate(ctx->args, iter); } taosHashCleanup(ctx->args); @@ -256,7 +256,7 @@ void transCtxMerge(STransCtx* dst, STransCtx* src) { STransCtxVal* dVal = taosHashGet(dst->args, key, klen); if (dVal) { - dVal->free(dVal->val); + dVal->freeFunc(dVal->val); } taosHashPut(dst->args, key, klen, sVal, sizeof(*sVal)); iter = taosHashIterate(src->args, iter); @@ -271,14 +271,14 @@ void* transCtxDumpVal(STransCtx* ctx, int32_t key) { if (cVal == NULL) { return NULL; } - char* ret = calloc(1, cVal->len); + char* ret = taosMemoryCalloc(1, cVal->len); memcpy(ret, (char*)cVal->val, cVal->len); return (void*)ret; } -void transQueueInit(STransQueue* queue, void (*free)(void* arg)) { +void transQueueInit(STransQueue* queue, void (*freeFunc)(const void* arg)) { queue->q = taosArrayInit(2, sizeof(void*)); - queue->free = free; + queue->freeFunc = freeFunc; } bool transQueuePush(STransQueue* queue, void* arg) { taosArrayPush(queue->q, &arg); @@ -308,10 +308,10 @@ bool transQueueEmpty(STransQueue* queue) { return taosArrayGetSize(queue->q) == 0; } void transQueueClear(STransQueue* queue) { - if (queue->free != NULL) { + if (queue->freeFunc != NULL) { for (int i = 0; i < taosArrayGetSize(queue->q); i++) { void* p = taosArrayGetP(queue->q, i); - queue->free(p); + queue->freeFunc(p); } } taosArrayClear(queue->q); diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index c6032a956980923bc00bd7e7a679c069e9ac32e6..d503c5b3d3269c7ecbb8e35669cf6acef6df098d 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -102,7 +102,7 @@ static const char* notify = "a"; tTrace("server conn %p received release request", conn); \ \ STransMsg tmsg = {.handle = (void*)conn, .code = 0}; \ - SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); \ + SSrvMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSrvMsg)); \ srvMsg->msg = tmsg; \ srvMsg->type = Release; \ srvMsg->pConn = conn; \ @@ -256,7 +256,7 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { } void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { buf->len = 2; - buf->base = calloc(1, sizeof(char) * buf->len); + buf->base = taosMemoryCalloc(1, sizeof(char) * buf->len); } void uvOnTimeoutCb(uv_timer_t* handle) { @@ -290,7 +290,7 @@ void uvOnSendCb(uv_write_t* req, int status) { memset(&conn->regArg, 0, sizeof(conn->regArg)); } transQueuePop(&conn->srvMsgs); - free(msg); + taosMemoryFree(msg); } else { uvStartSendRespInternal(msg); } @@ -308,7 +308,7 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) { } else { tError("fail to dispatch conn to work thread"); } - free(req); + taosMemoryFree(req); } static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { @@ -380,7 +380,7 @@ static void destroySmsg(SSrvMsg* smsg) { return; } transFreeMsg(smsg->msg.pCont); - free(smsg); + taosMemoryFree(smsg); } static void destroyAllConn(SWorkThrdObj* pThrd) { while (!QUEUE_IS_EMPTY(&pThrd->conn)) { @@ -430,7 +430,7 @@ static void uvShutDownCb(uv_shutdown_t* req, int status) { tDebug("conn failed to shut down: %s", uv_err_name(status)); } uv_close((uv_handle_t*)req->handle, uvDestroyConn); - free(req); + taosMemoryFree(req); } void uvOnAcceptCb(uv_stream_t* stream, int status) { @@ -439,11 +439,11 @@ void uvOnAcceptCb(uv_stream_t* stream, int status) { } SServerObj* pObj = container_of(stream, SServerObj, server); - uv_tcp_t* cli = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); + uv_tcp_t* cli = (uv_tcp_t*)taosMemoryMalloc(sizeof(uv_tcp_t)); uv_tcp_init(pObj->loop, cli); if (uv_accept(stream, (uv_stream_t*)cli) == 0) { - uv_write_t* wr = (uv_write_t*)malloc(sizeof(uv_write_t)); + uv_write_t* wr = (uv_write_t*)taosMemoryMalloc(sizeof(uv_write_t)); uv_buf_t buf = uv_buf_init((char*)notify, strlen(notify)); @@ -453,7 +453,7 @@ void uvOnAcceptCb(uv_stream_t* stream, int status) { uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, uvOnPipeWriteCb); } else { uv_close((uv_handle_t*)cli, NULL); - free(cli); + taosMemoryFree(cli); } } void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { @@ -469,7 +469,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { // free memory allocated by assert(nread == strlen(notify)); assert(buf->base[0] == notify[0]); - free(buf->base); + taosMemoryFree(buf->base); SWorkThrdObj* pThrd = q->data; @@ -492,7 +492,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { pConn->hostThrd = pThrd; // init client handle - pConn->pTcp = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); + pConn->pTcp = (uv_tcp_t*)taosMemoryMalloc(sizeof(uv_tcp_t)); uv_tcp_init(pThrd->loop, pConn->pTcp); pConn->pTcp->data = pConn; @@ -537,7 +537,7 @@ void* acceptThread(void* arg) { } static bool addHandleToWorkloop(void* arg) { SWorkThrdObj* pThrd = arg; - pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); + pThrd->loop = (uv_loop_t*)taosMemoryMalloc(sizeof(uv_loop_t)); if (0 != uv_loop_init(pThrd->loop)) { return false; } @@ -569,7 +569,7 @@ static bool addHandleToAcceptloop(void* arg) { } // register an async here to quit server gracefully - srv->pAcceptAsync = calloc(1, sizeof(uv_async_t)); + srv->pAcceptAsync = taosMemoryCalloc(1, sizeof(uv_async_t)); uv_async_init(srv->loop, srv->pAcceptAsync, uvAcceptAsyncCb); srv->pAcceptAsync->data = srv; @@ -596,7 +596,7 @@ void* workerThread(void* arg) { static SSrvConn* createConn(void* hThrd) { SWorkThrdObj* pThrd = hThrd; - SSrvConn* pConn = (SSrvConn*)calloc(1, sizeof(SSrvConn)); + SSrvConn* pConn = (SSrvConn*)taosMemoryCalloc(1, sizeof(SSrvConn)); QUEUE_INIT(&pConn->queue); QUEUE_PUSH(&pThrd->conn, &pConn->queue); @@ -621,7 +621,7 @@ static void destroyConn(SSrvConn* conn, bool clear) { transQueueDestroy(&conn->srvMsgs); if (clear) { tTrace("server conn %p to be destroyed", conn); - uv_shutdown_t* req = malloc(sizeof(uv_shutdown_t)); + uv_shutdown_t* req = taosMemoryMalloc(sizeof(uv_shutdown_t)); uv_shutdown(req, (uv_stream_t*)conn->pTcp, uvShutDownCb); } } @@ -635,8 +635,8 @@ static void uvDestroyConn(uv_handle_t* handle) { tDebug("server conn %p destroy", conn); uv_timer_stop(&conn->pTimer); QUEUE_REMOVE(&conn->queue); - free(conn->pTcp); - // free(conn); + taosMemoryFree(conn->pTcp); + // taosMemoryFree(conn); if (thrd->quit && QUEUE_IS_EMPTY(&thrd->conn)) { uv_loop_close(thrd->loop); @@ -645,22 +645,22 @@ static void uvDestroyConn(uv_handle_t* handle) { } void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { - SServerObj* srv = calloc(1, sizeof(SServerObj)); - srv->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); + SServerObj* srv = taosMemoryCalloc(1, sizeof(SServerObj)); + srv->loop = (uv_loop_t*)taosMemoryMalloc(sizeof(uv_loop_t)); srv->numOfThreads = numOfThreads; srv->workerIdx = 0; - srv->pThreadObj = (SWorkThrdObj**)calloc(srv->numOfThreads, sizeof(SWorkThrdObj*)); - srv->pipe = (uv_pipe_t**)calloc(srv->numOfThreads, sizeof(uv_pipe_t*)); + srv->pThreadObj = (SWorkThrdObj**)taosMemoryCalloc(srv->numOfThreads, sizeof(SWorkThrdObj*)); + srv->pipe = (uv_pipe_t**)taosMemoryCalloc(srv->numOfThreads, sizeof(uv_pipe_t*)); srv->ip = ip; srv->port = port; uv_loop_init(srv->loop); for (int i = 0; i < srv->numOfThreads; i++) { - SWorkThrdObj* thrd = (SWorkThrdObj*)calloc(1, sizeof(SWorkThrdObj)); + SWorkThrdObj* thrd = (SWorkThrdObj*)taosMemoryCalloc(1, sizeof(SWorkThrdObj)); thrd->quit = false; srv->pThreadObj[i] = thrd; - srv->pipe[i] = (uv_pipe_t*)calloc(2, sizeof(uv_pipe_t)); + srv->pipe[i] = (uv_pipe_t*)taosMemoryCalloc(2, sizeof(uv_pipe_t)); int fds[2]; if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { goto End; @@ -707,7 +707,7 @@ void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd) { destroyAllConn(thrd); thrd->quit = true; } - free(msg); + taosMemoryFree(msg); } void uvHandleRelease(SSrvMsg* msg, SWorkThrdObj* thrd) { // release handle to rpc init @@ -744,7 +744,7 @@ void uvHandleRegister(SSrvMsg* msg, SWorkThrdObj* thrd) { (*pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL); memset(&conn->regArg, 0, sizeof(conn->regArg)); } - free(msg); + taosMemoryFree(msg); } } void destroyWorkThrd(SWorkThrdObj* pThrd) { @@ -752,12 +752,12 @@ void destroyWorkThrd(SWorkThrdObj* pThrd) { return; } taosThreadJoin(pThrd->thread, NULL); - free(pThrd->loop); + taosMemoryFree(pThrd->loop); transDestroyAsyncPool(pThrd->asyncPool); - free(pThrd); + taosMemoryFree(pThrd); } void sendQuitToWorkThrd(SWorkThrdObj* pThrd) { - SSrvMsg* msg = calloc(1, sizeof(SSrvMsg)); + SSrvMsg* msg = taosMemoryCalloc(1, sizeof(SSrvMsg)); msg->type = Quit; tDebug("server send quit msg to work thread"); transSendAsync(pThrd->asyncPool, &msg->q); @@ -775,16 +775,16 @@ void transCloseServer(void* arg) { uv_async_send(srv->pAcceptAsync); taosThreadJoin(srv->thread, NULL); - free(srv->pThreadObj); - free(srv->pAcceptAsync); - free(srv->loop); + taosMemoryFree(srv->pThreadObj); + taosMemoryFree(srv->pAcceptAsync); + taosMemoryFree(srv->loop); for (int i = 0; i < srv->numOfThreads; i++) { - free(srv->pipe[i]); + taosMemoryFree(srv->pipe[i]); } - free(srv->pipe); + taosMemoryFree(srv->pipe); - free(srv); + taosMemoryFree(srv); } void transRefSrvHandle(void* handle) { @@ -817,7 +817,7 @@ void transReleaseSrvHandle(void* handle) { STransMsg tmsg = {.handle = handle, .code = 0}; - SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); + SSrvMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSrvMsg)); srvMsg->msg = tmsg; srvMsg->type = Release; srvMsg->pConn = pConn; @@ -832,7 +832,7 @@ void transSendResponse(const STransMsg* pMsg) { SSrvConn* pConn = pMsg->handle; SWorkThrdObj* pThrd = pConn->hostThrd; - SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); + SSrvMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSrvMsg)); srvMsg->pConn = pConn; srvMsg->msg = *pMsg; srvMsg->type = Normal; @@ -846,7 +846,7 @@ void transRegisterMsg(const STransMsg* msg) { SSrvConn* pConn = msg->handle; SWorkThrdObj* pThrd = pConn->hostThrd; - SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); + SSrvMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSrvMsg)); srvMsg->pConn = pConn; srvMsg->msg = *msg; srvMsg->type = Register; diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index 4af316acae25b833c165656aa00cc2dd50ddb750..7d3c3aa012465b5fac8879ebd9e480a74895778b 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -184,7 +184,7 @@ int main(int argc, char *argv[]) { taosGetTimeOfDay(&systemTime); startTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; - SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo) * appThreads); + SInfo *pInfo = (SInfo *)taosMemoryCalloc(1, sizeof(SInfo) * appThreads); taosThreadAttrInit(&thattr); taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); diff --git a/source/libs/transport/test/rsclient.c b/source/libs/transport/test/rsclient.c index f3be4e34524ae13f86ca92b9bc60e0b92f032fc6..e7fc7b45e1ed74b3a8d82983ba381757cadfc831 100644 --- a/source/libs/transport/test/rsclient.c +++ b/source/libs/transport/test/rsclient.c @@ -161,7 +161,7 @@ int main(int argc, char *argv[]) { taosGetTimeOfDay(&systemTime); startTime = systemTime.tv_sec*1000000 + systemTime.tv_usec; - SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo)*appThreads); + SInfo *pInfo = (SInfo *)taosMemoryCalloc(1, sizeof(SInfo)*appThreads); taosThreadAttrInit(&thattr); taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); diff --git a/source/libs/transport/test/syncClient.c b/source/libs/transport/test/syncClient.c index 14f32df0b8034a8c54b42583d9f9c380fbb11a70..27a468a86fb7e3f011d5826ef524fe5bfb55aa6d 100644 --- a/source/libs/transport/test/syncClient.c +++ b/source/libs/transport/test/syncClient.c @@ -184,7 +184,7 @@ int main(int argc, char *argv[]) { taosGetTimeOfDay(&systemTime); startTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; - SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo) * appThreads); + SInfo *pInfo = (SInfo *)taosMemoryCalloc(1, sizeof(SInfo) * appThreads); taosThreadAttrInit(&thattr); taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); diff --git a/source/libs/transport/test/transportTests.cpp b/source/libs/transport/test/transportTests.cpp index 65d930299423963f2793d229384a1131d94762bc..90122eadd1353246d99f28ccc635934ea5a9b2d7 100644 --- a/source/libs/transport/test/transportTests.cpp +++ b/source/libs/transport/test/transportTests.cpp @@ -90,7 +90,7 @@ TEST_F(QueueEnv, testPushAndPop) { assert(q->IsEmpty()); for (int i = 0; i < 100; i++) { - QueueElem *el = (QueueElem *)malloc(sizeof(QueueElem)); + QueueElem *el = (QueueElem *)taosMemoryMalloc(sizeof(QueueElem)); el->val = i; q->Push(el); } @@ -98,7 +98,7 @@ TEST_F(QueueEnv, testPushAndPop) { while (!q->IsEmpty()) { QueueElem *el = q->Pop(); assert(el->val == i++); - free(el); + taosMemoryFree(el); } assert(q->IsEmpty()); } @@ -109,7 +109,7 @@ TEST_F(QueueEnv, testRm) { assert(q->IsEmpty()); for (int i = 0; i < 100; i++) { - QueueElem *el = (QueueElem *)malloc(sizeof(QueueElem)); + QueueElem *el = (QueueElem *)taosMemoryMalloc(sizeof(QueueElem)); el->val = i; q->Push(el); set.push_back(el); @@ -117,7 +117,7 @@ TEST_F(QueueEnv, testRm) { for (int i = set.size() - 1; i >= 0; i--) { QueueElem *el = set[i]; q->RmElem(el); - free(el); + taosMemoryFree(el); } assert(q->IsEmpty()); } @@ -126,7 +126,7 @@ TEST_F(QueueEnv, testIter) { assert(q->IsEmpty()); std::vector vals; for (int i = 0; i < 100; i++) { - QueueElem *el = (QueueElem *)malloc(sizeof(QueueElem)); + QueueElem *el = (QueueElem *)taosMemoryMalloc(sizeof(QueueElem)); el->val = i; q->Push(el); vals.push_back(i); @@ -139,7 +139,7 @@ TEST_F(QueueEnv, testIter) { class TransCtxEnv : public ::testing::Test { protected: virtual void SetUp() { - ctx = (STransCtx *)calloc(1, sizeof(STransCtx)); + ctx = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx)); transCtxInit(ctx); // TODO } @@ -153,57 +153,57 @@ class TransCtxEnv : public ::testing::Test { TEST_F(TransCtxEnv, mergeTest) { int key = 1; { - STransCtx *src = (STransCtx *)calloc(1, sizeof(STransCtx)); + STransCtx *src = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx)); transCtxInit(src); { - STransCtxVal val1 = {.val = NULL, .len = 0, .free = free}; - val1.val = malloc(12); + STransCtxVal val1 = {.val = NULL, .len = 0, .freeFunc = taosMemoryFree}; + val1.val = taosMemoryMalloc(12); val1.len = 12; taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); key++; } { - STransCtxVal val1 = {.val = NULL, .len = 0, .free = free}; - val1.val = malloc(12); + STransCtxVal val1 = {.val = NULL, .len = 0, .freeFunc = taosMemoryFree}; + val1.val = taosMemoryMalloc(12); val1.len = 12; taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); key++; } transCtxMerge(ctx, src); - free(src); + taosMemoryFree(src); } EXPECT_EQ(2, taosHashGetSize(ctx->args)); { - STransCtx *src = (STransCtx *)calloc(1, sizeof(STransCtx)); + STransCtx *src = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx)); transCtxInit(src); { - STransCtxVal val1 = {.val = NULL, .len = 0, .free = free}; - val1.val = malloc(12); + STransCtxVal val1 = {.val = NULL, .len = 0, .freeFunc = taosMemoryFree}; + val1.val = taosMemoryMalloc(12); val1.len = 12; taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); key++; } { - STransCtxVal val1 = {.val = NULL, .len = 0, .free = free}; - val1.val = malloc(12); + STransCtxVal val1 = {.val = NULL, .len = 0, .freeFunc = taosMemoryFree}; + val1.val = taosMemoryMalloc(12); val1.len = 12; taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); key++; } transCtxMerge(ctx, src); - free(src); + taosMemoryFree(src); } std::string val("Hello"); EXPECT_EQ(4, taosHashGetSize(ctx->args)); { key = 1; - STransCtx *src = (STransCtx *)calloc(1, sizeof(STransCtx)); + STransCtx *src = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx)); transCtxInit(src); { - STransCtxVal val1 = {.val = NULL, .len = 0, .free = free}; - val1.val = calloc(1, 11); + STransCtxVal val1 = {.val = NULL, .len = 0, .freeFunc = taosMemoryFree}; + val1.val = taosMemoryCalloc(1, 11); memcpy(val1.val, val.c_str(), val.size()); val1.len = 11; @@ -211,21 +211,21 @@ TEST_F(TransCtxEnv, mergeTest) { key++; } { - STransCtxVal val1 = {.val = NULL, .len = 0, .free = free}; - val1.val = calloc(1, 11); + STransCtxVal val1 = {.val = NULL, .len = 0, .freeFunc = taosMemoryFree}; + val1.val = taosMemoryCalloc(1, 11); memcpy(val1.val, val.c_str(), val.size()); val1.len = 11; taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); key++; } transCtxMerge(ctx, src); - free(src); + taosMemoryFree(src); } EXPECT_EQ(4, taosHashGetSize(ctx->args)); char *skey = (char *)transCtxDumpVal(ctx, 1); EXPECT_EQ(0, strcmp(skey, val.c_str())); - free(skey); + taosMemoryFree(skey); skey = (char *)transCtxDumpVal(ctx, 2); EXPECT_EQ(0, strcmp(skey, val.c_str())); diff --git a/source/libs/transport/test/uv.c b/source/libs/transport/test/uv.c index 5a19b7998c915343792cfc4051358eb763b0f863..fb026ef1a61fc190eb1b560520a83809191023d5 100644 --- a/source/libs/transport/test/uv.c +++ b/source/libs/transport/test/uv.c @@ -38,7 +38,7 @@ void echo_write(uv_write_t *req, int status) { fprintf(stderr, "Write error %s\n", uv_err_name(status)); } printf("write data to client\n"); - free(req); + taosMemoryFree(req); } void echo_read(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { @@ -47,14 +47,14 @@ void echo_read(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { pConn->ref += 1; printf("read data %d\n", nread, buf->base, buf->len); if (nread > 0) { - uv_write_t *req = (uv_write_t *)malloc(sizeof(uv_write_t)); + uv_write_t *req = (uv_write_t *)taosMemoryMalloc(sizeof(uv_write_t)); // dispatch request to database other process thread // just write out uv_buf_t write_out; write_out.base = buf->base; write_out.len = nread; uv_write((uv_write_t *)req, client, &write_out, 1, echo_write); - free(buf->base); + taosMemoryFree(buf->base); return; } @@ -63,11 +63,11 @@ void echo_read(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { fprintf(stderr, "Read error %s\n", uv_err_name(nread)); uv_close((uv_handle_t *)client, NULL); } - free(buf->base); + taosMemoryFree(buf->base); } void alloc_buffer(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf) { - buf->base = malloc(suggested_size); + buf->base = taosMemoryMalloc(suggested_size); buf->len = suggested_size; } @@ -79,10 +79,10 @@ void on_new_connection(uv_stream_t *s, int status) { SServerObj *pObj = container_of(s, SServerObj, server); printf("new_connection from client\n"); - uv_tcp_t *client = (uv_tcp_t *)malloc(sizeof(uv_tcp_t)); + uv_tcp_t *client = (uv_tcp_t *)taosMemoryMalloc(sizeof(uv_tcp_t)); uv_tcp_init(pObj->loop, client); if (uv_accept(s, (uv_stream_t *)client) == 0) { - uv_write_t *write_req = (uv_write_t *)malloc(sizeof(uv_write_t)); + uv_write_t *write_req = (uv_write_t *)taosMemoryMalloc(sizeof(uv_write_t)); uv_buf_t dummy_buf = uv_buf_init("a", 1); // despatch to worker thread pObj->workerIdx = (pObj->workerIdx + 1) % pObj->numOfThread; @@ -112,13 +112,13 @@ void child_on_new_connection(uv_stream_t *q, ssize_t nread, uv_handle_type pending = uv_pipe_pending_type(pipe); assert(pending == UV_TCP); - SConnCtx *pConn = malloc(sizeof(SConnCtx)); + SConnCtx *pConn = taosMemoryMalloc(sizeof(SConnCtx)); /* init conn timer*/ - pConn->pTimer = malloc(sizeof(uv_timer_t)); + pConn->pTimer = taosMemoryMalloc(sizeof(uv_timer_t)); uv_timer_init(pObj->loop, pConn->pTimer); - pConn->pClient = (uv_tcp_t *)malloc(sizeof(uv_tcp_t)); + pConn->pClient = (uv_tcp_t *)taosMemoryMalloc(sizeof(uv_tcp_t)); pConn->pWorkerAsync = pObj->workerAsync; // thread safty uv_tcp_init(pObj->loop, pConn->pClient); @@ -130,10 +130,10 @@ void child_on_new_connection(uv_stream_t *q, ssize_t nread, uv_read_start((uv_stream_t *)(pConn->pClient), alloc_buffer, echo_read); } else { uv_timer_stop(pConn->pTimer); - free(pConn->pTimer); + taosMemoryFree(pConn->pTimer); uv_close((uv_handle_t *)pConn->pClient, NULL); - free(pConn->pClient); - free(pConn); + taosMemoryFree(pConn->pClient); + taosMemoryFree(pConn); } } @@ -144,13 +144,13 @@ static void workerAsyncCallback(uv_async_t *handle) { void *worker_thread(void *arg) { SThreadObj *pObj = (SThreadObj *)arg; int fd = pObj->fd; - pObj->loop = (uv_loop_t *)malloc(sizeof(uv_loop_t)); + pObj->loop = (uv_loop_t *)taosMemoryMalloc(sizeof(uv_loop_t)); uv_loop_init(pObj->loop); uv_pipe_init(pObj->loop, pObj->pipe, 1); uv_pipe_open(pObj->pipe, fd); - pObj->workerAsync = malloc(sizeof(uv_async_t)); + pObj->workerAsync = taosMemoryMalloc(sizeof(uv_async_t)); uv_async_init(pObj->loop, pObj->workerAsync, workerAsyncCallback); uv_read_start((uv_stream_t *)pObj->pipe, alloc_buffer, child_on_new_connection); @@ -159,19 +159,19 @@ void *worker_thread(void *arg) { } int main() { - SServerObj *server = calloc(1, sizeof(SServerObj)); - server->loop = (uv_loop_t *)malloc(sizeof(uv_loop_t)); + SServerObj *server = taosMemoryCalloc(1, sizeof(SServerObj)); + server->loop = (uv_loop_t *)taosMemoryMalloc(sizeof(uv_loop_t)); server->numOfThread = NUM_OF_THREAD; server->workerIdx = 0; server->pThreadObj = - (SThreadObj **)calloc(server->numOfThread, sizeof(SThreadObj *)); - server->pipe = (uv_pipe_t **)calloc(server->numOfThread, sizeof(uv_pipe_t *)); + (SThreadObj **)taosMemoryCalloc(server->numOfThread, sizeof(SThreadObj *)); + server->pipe = (uv_pipe_t **)taosMemoryCalloc(server->numOfThread, sizeof(uv_pipe_t *)); uv_loop_init(server->loop); for (int i = 0; i < server->numOfThread; i++) { - server->pThreadObj[i] = (SThreadObj *)calloc(1, sizeof(SThreadObj)); - server->pipe[i] = (uv_pipe_t *)calloc(2, sizeof(uv_pipe_t)); + server->pThreadObj[i] = (SThreadObj *)taosMemoryCalloc(1, sizeof(SThreadObj)); + server->pipe[i] = (uv_pipe_t *)taosMemoryCalloc(2, sizeof(uv_pipe_t)); int fds[2]; if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 70ec7e06557d75358f2b4e33a8aec06b493e06d2..cbe7cb81db9f855597bc717e6bfe441b281bcc11 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -74,7 +74,7 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { uint64_t magic = WAL_MAGIC; - char* buf = malloc(readSize + 5); + char* buf = taosMemoryMalloc(readSize + 5); if (buf == NULL) { taosCloseFile(&pFile); terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; @@ -83,7 +83,7 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { taosLSeekFile(pFile, -readSize, SEEK_END); if (readSize != taosReadFile(pFile, buf, readSize)) { - free(buf); + taosMemoryFree(buf); taosCloseFile(&pFile); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -104,7 +104,7 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { SWalHead *logContent = (SWalHead*)found; if (walValidHeadCksum(logContent) != 0 || walValidBodyCksum(logContent) != 0) { // file has to be deleted - free(buf); + taosMemoryFree(buf); taosCloseFile(&pFile); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; @@ -215,7 +215,7 @@ int walRollFileInfo(SWal* pWal) { } // TODO: change to emplace back - SWalFileInfo* pNewInfo = malloc(sizeof(SWalFileInfo)); + SWalFileInfo* pNewInfo = taosMemoryMalloc(sizeof(SWalFileInfo)); if (pNewInfo == NULL) { terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; return -1; @@ -226,7 +226,7 @@ int walRollFileInfo(SWal* pWal) { pNewInfo->closeTs = -1; pNewInfo->fileSize = 0; taosArrayPush(pArray, pNewInfo); - free(pNewInfo); + taosMemoryFree(pNewInfo); return 0; } @@ -378,7 +378,7 @@ int walSaveMeta(SWal* pWal) { walBuildMetaName(pWal, metaVer, fnameStr); taosRemoveFile(fnameStr); } - free(serialized); + taosMemoryFree(serialized); return 0; } @@ -395,7 +395,7 @@ int walLoadMeta(SWal* pWal) { int64_t file_size = 0; taosStatFile(fnameStr, &file_size, NULL); int size = (int)file_size; - char* buf = malloc(size + 5); + char* buf = taosMemoryMalloc(size + 5); if (buf == NULL) { terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; return -1; @@ -409,12 +409,12 @@ int walLoadMeta(SWal* pWal) { if (taosReadFile(pFile, buf, size) != size) { terrno = TAOS_SYSTEM_ERROR(errno); taosCloseFile(&pFile); - free(buf); + taosMemoryFree(buf); return -1; } // load into fileInfoSet int code = walMetaDeserialize(pWal, buf); taosCloseFile(&pFile); - free(buf); + taosMemoryFree(buf); return code; } diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 2da8f4f8af46ff9156ec61ab5de3d42f2bfc4f4c..cf40c998deabb1b869b8e64cbd84b7f52dbb01ea 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -63,7 +63,7 @@ void walCleanUp() { } SWal *walOpen(const char *path, SWalCfg *pCfg) { - SWal *pWal = malloc(sizeof(SWal)); + SWal *pWal = taosMemoryMalloc(sizeof(SWal)); if (pWal == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; @@ -88,7 +88,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo)); if (pWal->fileInfoSet == NULL) { wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno)); - free(pWal); + taosMemoryFree(pWal); return NULL; } @@ -103,7 +103,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { if (taosThreadMutexInit(&pWal->mutex, NULL) < 0) { taosArrayDestroy(pWal->fileInfoSet); - free(pWal); + taosMemoryFree(pWal); return NULL; } @@ -111,7 +111,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { if (pWal->refId < 0) { taosThreadMutexDestroy(&pWal->mutex); taosArrayDestroy(pWal->fileInfoSet); - free(pWal); + taosMemoryFree(pWal); return NULL; } @@ -121,7 +121,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { taosRemoveRef(tsWal.refSetId, pWal->refId); taosThreadMutexDestroy(&pWal->mutex); taosArrayDestroy(pWal->fileInfoSet); - free(pWal); + taosMemoryFree(pWal); return NULL; } @@ -174,7 +174,7 @@ static void walFreeObj(void *wal) { wDebug("vgId:%d, wal:%p is freed", pWal->cfg.vgId, pWal); taosThreadMutexDestroy(&pWal->mutex); - tfree(pWal); + taosMemoryFreeClear(pWal); } static bool walNeedFsync(SWal *pWal) { diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 8d3acef9e74646629284d96a783847088525a1f1..e15c1620480c1825714ab7d36550b31a5be9132f 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -17,7 +17,7 @@ #include "taoserror.h" SWalReadHandle *walOpenReadHandle(SWal *pWal) { - SWalReadHandle *pRead = malloc(sizeof(SWalReadHandle)); + SWalReadHandle *pRead = taosMemoryMalloc(sizeof(SWalReadHandle)); if (pRead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -30,10 +30,10 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) { pRead->curFileFirstVer = -1; pRead->capacity = 0; pRead->status = 0; - pRead->pHead = malloc(sizeof(SWalHead)); + pRead->pHead = taosMemoryMalloc(sizeof(SWalHead)); if (pRead->pHead == NULL) { terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; - free(pRead); + taosMemoryFree(pRead); return NULL; } return pRead; @@ -42,8 +42,8 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) { void walCloseReadHandle(SWalReadHandle *pRead) { taosCloseFile(&pRead->pReadIdxTFile); taosCloseFile(&pRead->pReadLogTFile); - tfree(pRead->pHead); - free(pRead); + taosMemoryFreeClear(pRead->pHead); + taosMemoryFree(pRead); } int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { return 0; } @@ -156,7 +156,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { return -1; } if (pRead->capacity < pRead->pHead->head.len) { - void *ptr = realloc(pRead->pHead, sizeof(SWalHead) + pRead->pHead->head.len); + void *ptr = taosMemoryRealloc(pRead->pHead, sizeof(SWalHead) + pRead->pHead->head.len); if (ptr == NULL) { terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; return -1; @@ -195,7 +195,7 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { return code; } if (*ppHead == NULL) { - void *ptr = realloc(*ppHead, sizeof(SWalHead)); + void *ptr = taosMemoryRealloc(*ppHead, sizeof(SWalHead)); if (ptr == NULL) { return -1; } @@ -208,9 +208,9 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { if (walValidHeadCksum(*ppHead) != 0) { return -1; } - void *ptr = realloc(*ppHead, sizeof(SWalHead) + (*ppHead)->head.len); + void *ptr = taosMemoryRealloc(*ppHead, sizeof(SWalHead) + (*ppHead)->head.len); if (ptr == NULL) { - free(*ppHead); + taosMemoryFree(*ppHead); *ppHead = NULL; return -1; } diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index 69ae83154acc7bbdbedc1f05654bbf8993c5c5e7..b1de3a1dcee592d5e69dd40504008f9a100647bc 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -19,7 +19,7 @@ class WalCleanEnv : public ::testing::Test { void SetUp() override { taosRemoveDir(pathName); - SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); + SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); memset(pCfg, 0, sizeof(SWalCfg)); pCfg->rollPeriod = -1; pCfg->segSize = -1; @@ -27,7 +27,7 @@ class WalCleanEnv : public ::testing::Test { pCfg->retentionSize = 0; pCfg->level = TAOS_WAL_FSYNC; pWal = walOpen(pathName, pCfg); - free(pCfg); + taosMemoryFree(pCfg); ASSERT(pWal != NULL); } @@ -51,13 +51,13 @@ class WalCleanDeleteEnv : public ::testing::Test { void SetUp() override { taosRemoveDir(pathName); - SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); + SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); memset(pCfg, 0, sizeof(SWalCfg)); pCfg->retentionPeriod = 0; pCfg->retentionSize = 0; pCfg->level = TAOS_WAL_FSYNC; pWal = walOpen(pathName, pCfg); - free(pCfg); + taosMemoryFree(pCfg); ASSERT(pWal != NULL); } @@ -86,7 +86,7 @@ class WalKeepEnv : public ::testing::Test { } void SetUp() override { - SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); + SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); memset(pCfg, 0, sizeof(SWalCfg)); pCfg->rollPeriod = -1; pCfg->segSize = -1; @@ -94,7 +94,7 @@ class WalKeepEnv : public ::testing::Test { pCfg->retentionSize = 0; pCfg->level = TAOS_WAL_FSYNC; pWal = walOpen(pathName, pCfg); - free(pCfg); + taosMemoryFree(pCfg); ASSERT(pWal != NULL); } @@ -172,7 +172,7 @@ TEST_F(WalCleanEnv, serialize) { ASSERT(code == 0); char* ss = walMetaSerialize(pWal); printf("%s\n", ss); - free(ss); + taosMemoryFree(ss); code = walSaveMeta(pWal); ASSERT(code == 0); } @@ -216,8 +216,8 @@ TEST_F(WalKeepEnv, readOldMeta) { for (int i = 0; i < len; i++) { EXPECT_EQ(oldss[i], newss[i]); } - free(oldss); - free(newss); + taosMemoryFree(oldss); + taosMemoryFree(newss); } TEST_F(WalCleanEnv, write) { diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 4dece8abefa6c0d25d0dfd0376590a47506d6538..ed93708c0775793b768f8339d33fa82bc571cb31 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -268,7 +268,7 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { autoDelFileListAdd(path); } - TdFilePtr pFile = (TdFilePtr)malloc(sizeof(TdFile)); + TdFilePtr pFile = (TdFilePtr)taosMemoryMalloc(sizeof(TdFile)); if (pFile == NULL) { if (fd >= 0) close(fd); if (fp != NULL) fclose(fp); @@ -312,7 +312,7 @@ int64_t taosCloseFile(TdFilePtr *ppFile) { taosThreadRwlockUnlock(&((*ppFile)->rwlock)); taosThreadRwlockDestroy(&((*ppFile)->rwlock)); #endif - free(*ppFile); + taosMemoryFree(*ppFile); *ppFile = NULL; return 0; #endif diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c index 5f12f9cd3d5a2a75cd4080d1f35cdb892ea8e0b8..2df61c08d418788ecadcdff7e869a948d8c6855e 100644 --- a/source/os/src/osLocale.c +++ b/source/os/src/osLocale.c @@ -134,7 +134,7 @@ void taosGetSystemLocale(char *outLocale, char *outCharset) { char *revisedCharset = taosCharsetReplace(str); tstrncpy(outCharset, revisedCharset, TD_CHARSET_LEN); - free(revisedCharset); + taosMemoryFree(revisedCharset); // printf("charset not configured, set to system default:%s", outCharset); } else { strcpy(outCharset, "UTF-8"); @@ -179,7 +179,7 @@ void taosGetSystemLocale(char *outLocale, char *outCharset) { char *revisedCharset = taosCharsetReplace(str); tstrncpy(outCharset, revisedCharset, TD_LOCALE_LEN); - free(revisedCharset); + taosMemoryFree(revisedCharset); // printf("charset not configured, set to system default:%s", outCharset); } else { strcpy(outCharset, "UTF-8"); diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c new file mode 100644 index 0000000000000000000000000000000000000000..2eb176b767b2c6543d5a7b12d6d44f0544033c4b --- /dev/null +++ b/source/os/src/osMemory.c @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define ALLOW_FORBID_FUNC +#include "os.h" + +#define USE_TD_MEMORY + +#define TD_MEMORY_SYMBOL ('T'<<24|'A'<<16|'O'<<8|'S') + +#define TEST_SIZE 4 + +typedef struct TdMemoryInfo +{ + int32_t symbol; + char **stackTrace; + int32_t stackTraceDepth; + int32_t memorySize; +} *TdMemoryInfoPtr , TdMemoryInfo; + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + +#else + +#include +#define STACKCALL __attribute__((regparm(1), noinline)) +void **STACKCALL taosGetEBP(void) { + void **ebp = NULL; + __asm__ __volatile__("mov %%rbp, %0;\n\t" + : "=m"(ebp) /* output */ + : /* input */ + : "memory"); /* not affect register */ + return (void **)(*ebp); +} +int32_t taosBackTrace(void **buffer, int32_t size) { + int32_t frame = 0; + void **ebp; + void **ret = NULL; + unsigned long long func_frame_distance = 0; + if (buffer != NULL && size > 0) { + ebp = taosGetEBP(); + func_frame_distance = (unsigned long long)(*ebp) - (unsigned long long)ebp; + while (ebp && frame < size && (func_frame_distance < (1ULL << 24)) // assume function ebp more than 16M + && (func_frame_distance > 0)) { + ret = ebp + 1; + buffer[frame++] = *ret; + ebp = (void **)(*ebp); + func_frame_distance = (unsigned long long)(*ebp) - (unsigned long long)ebp; + } + } + return frame; +} +#endif + +char **taosBackTraceSymbols(int32_t *size) { + void *buffer[20] = {NULL}; + *size = taosBackTrace(buffer, 10); + return backtrace_symbols(buffer, *size); +} + +void *taosMemoryMalloc(int32_t size) { + void *tmp = malloc(size + sizeof(TdMemoryInfo)); + if (tmp == NULL) return NULL; + + TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)tmp; + pTdMemoryInfo->memorySize = size; + pTdMemoryInfo->symbol = TD_MEMORY_SYMBOL; + pTdMemoryInfo->stackTrace = taosBackTraceSymbols(&pTdMemoryInfo->stackTraceDepth); + + return (char*)tmp + sizeof(TdMemoryInfo); +} + +void *taosMemoryCalloc(int32_t num, int32_t size) { + int32_t memorySize = num * size; + char *tmp = calloc(memorySize + sizeof(TdMemoryInfo), 1); + if (tmp == NULL) return NULL; + + TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)tmp; + pTdMemoryInfo->memorySize = memorySize; + pTdMemoryInfo->symbol = TD_MEMORY_SYMBOL; + pTdMemoryInfo->stackTrace = taosBackTraceSymbols(&pTdMemoryInfo->stackTraceDepth); + + return (char*)tmp + sizeof(TdMemoryInfo); +} + +void *taosMemoryRealloc(void *ptr, int32_t size) { + if (ptr == NULL) return taosMemoryMalloc(size); + + TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char*)ptr - sizeof(TdMemoryInfo)); + assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); + + TdMemoryInfo tdMemoryInfo; + memcpy(&tdMemoryInfo, pTdMemoryInfo, sizeof(TdMemoryInfo)); + + void *tmp = realloc(pTdMemoryInfo, size + sizeof(TdMemoryInfo)); + if (tmp == NULL) return NULL; + + memcpy(tmp, &tdMemoryInfo, sizeof(TdMemoryInfo)); + ((TdMemoryInfoPtr)tmp)->memorySize = size; + + return (char*)tmp + sizeof(TdMemoryInfo); +} + +void taosMemoryFree(const void *ptr) { + if (ptr == NULL) return; + + TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char*)ptr - sizeof(TdMemoryInfo)); + if(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL) { + if(pTdMemoryInfo->stackTrace != NULL) free(pTdMemoryInfo->stackTrace); + memset(pTdMemoryInfo, 0, sizeof(TdMemoryInfo)); + free(pTdMemoryInfo); + } else { + free((void*)ptr); + } +} + +int32_t taosMemorySize(void *ptr) { + if (ptr == NULL) return 0; + + TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char*)ptr - sizeof(TdMemoryInfo)); + assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); + + return pTdMemoryInfo->memorySize; +} \ No newline at end of file diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index b41ef898d40fd38c64501378b568f1b7dbc62bf9..1297fdbc276cc3a9a4611fa296ba495b452287e3 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -143,7 +143,7 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) { fprintf(stderr, "==%s[%d]%s():[%p]==already initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } - struct tsem_s *p = (struct tsem_s *)calloc(1, sizeof(*p)); + struct tsem_s *p = (struct tsem_s *)taosMemoryCalloc(1, sizeof(*p)); if (!p) { fprintf(stderr, "==%s[%d]%s():[%p]==out of memory\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); @@ -331,7 +331,7 @@ int tsem_destroy(tsem_t *sem) { #endif // SEM_USE_PTHREAD p->valid = 0; - free(p); + taosMemoryFree(p); *sem = NULL; return 0; diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 0e6abb57853e23954b6c55a0419b16fc24abec30..b130689d4b5aa7bba55d341796cf6416594209d5 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -47,9 +47,6 @@ #endif #endif -typedef int32_t SocketFd; -typedef SocketFd EpollFd; - typedef struct TdSocketServer { #if SOCKET_WITH_LOCK TdThreadRwlock rwlock; @@ -58,14 +55,6 @@ typedef struct TdSocketServer { SocketFd fd; } *TdSocketServerPtr, TdSocketServer; -typedef struct TdSocket { -#if SOCKET_WITH_LOCK - TdThreadRwlock rwlock; -#endif - int refId; - SocketFd fd; -} *TdSocketPtr, TdSocket; - typedef struct TdEpoll { #if SOCKET_WITH_LOCK TdThreadRwlock rwlock; @@ -128,7 +117,7 @@ int32_t taosCloseSocket(TdSocketPtr *ppSocket) { } code = taosCloseSocketNoCheck1((*ppSocket)->fd); (*ppSocket)->fd = -1; - free(*ppSocket); + taosMemoryFree(*ppSocket); return code; } int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer) { @@ -138,7 +127,7 @@ int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer) { } code = taosCloseSocketNoCheck1((*ppSocketServer)->fd); (*ppSocketServer)->fd = -1; - free(*ppSocketServer); + taosMemoryFree(*ppSocketServer); return code; } @@ -451,7 +440,7 @@ TdSocketPtr taosOpenUdpSocket(uint32_t ip, uint16_t port) { return NULL; } - TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); if (pSocket == NULL) { taosCloseSocketNoCheck1(fd); return NULL; @@ -495,7 +484,7 @@ TdSocketPtr taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t return NULL; } - TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); if (pSocket == NULL) { taosCloseSocketNoCheck1(fd); return NULL; @@ -665,7 +654,7 @@ TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { return NULL; } - TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); if (pSocket == NULL) { taosCloseSocketNoCheck1(fd); return NULL; @@ -714,7 +703,7 @@ TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct s return NULL; } - TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); if (pSocket == NULL) { taosCloseSocketNoCheck1(fd); return NULL; @@ -912,7 +901,7 @@ TdEpollPtr taosCreateEpoll(int32_t size) { return NULL; } - TdEpollPtr pEpoll = (TdEpollPtr)malloc(sizeof(TdEpoll)); + TdEpollPtr pEpoll = (TdEpollPtr)taosMemoryMalloc(sizeof(TdEpoll)); if (pEpoll == NULL) { taosCloseSocketNoCheck1(fd); return NULL; @@ -950,6 +939,6 @@ int32_t taosCloseEpoll(TdEpollPtr *ppEpoll) { } code = taosCloseSocketNoCheck1((*ppEpoll)->fd); (*ppEpoll)->fd = -1; - free(*ppEpoll); + taosMemoryFree(*ppEpoll); return code; } diff --git a/source/os/src/osString.c b/source/os/src/osString.c index d3d1ab5ddaf0c97ce6d2d8e0f5037509721e5a00..5adb564e52f0d050fe7885f21d8dd11349518763 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -15,7 +15,6 @@ #define ALLOW_FORBID_FUNC #define _DEFAULT_SOURCE -#include #include "os.h" #ifndef DISALLOW_NCHAR_WITHOUT_ICONV @@ -50,8 +49,8 @@ int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) { #if 0 int32_t ucs4_max_len = bytes + 4; - char *f1_mbs = calloc(bytes, 1); - char *f2_mbs = calloc(bytes, 1); + char *f1_mbs = taosMemoryCalloc(bytes, 1); + char *f2_mbs = taosMemoryCalloc(bytes, 1); if (taosUcs4ToMbs(f1_ucs4, ucs4_max_len, f1_mbs) < 0) { return -1; } @@ -59,15 +58,15 @@ int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) { return -1; } int32_t ret = strcmp(f1_mbs, f2_mbs); - free(f1_mbs); - free(f2_mbs); + taosMemoryFree(f1_mbs); + taosMemoryFree(f2_mbs); return ret; #endif } TdUcs4* tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4) { - assert(malloc_usable_size(target_ucs4)>=len_ucs4*sizeof(TdUcs4)); + assert(taosMemorySize(target_ucs4)>=len_ucs4*sizeof(TdUcs4)); return memcpy(target_ucs4, source_ucs4, len_ucs4*sizeof(TdUcs4)); } diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 9bff7605097f20794b9a2d7b89e1127dc3a7d3b7..eb5712d2e541d2827f115b1e3bddc7456608934b 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -145,7 +145,7 @@ static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle); - if (line != NULL) tfree(line); + if (line != NULL) taosMemoryFreeClear(line); taosCloseFile(&pFile); return 0; } @@ -174,7 +174,7 @@ static int32_t taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { } } - if (line != NULL) tfree(line); + if (line != NULL) taosMemoryFreeClear(line); taosCloseFile(&pFile); return 0; } @@ -268,7 +268,7 @@ int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { } } - if (line != NULL) free(line); + if (line != NULL) taosMemoryFree(line); taosCloseFile(&pFile); return code; #else @@ -293,7 +293,7 @@ int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { } } - if (line != NULL) free(line); + if (line != NULL) taosMemoryFree(line); taosCloseFile(&pFile); return code; #endif @@ -324,7 +324,7 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { } } - if (line != NULL) free(line); + if (line != NULL) taosMemoryFree(line); taosCloseFile(&pFile); return code; @@ -351,7 +351,7 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { } } - if (line != NULL) free(line); + if (line != NULL) taosMemoryFree(line); taosCloseFile(&pFile); return code; @@ -486,7 +486,7 @@ int32_t taosGetProcMemory(int64_t *usedKB) { char tmp[10]; sscanf(line, "%s %" PRId64, tmp, usedKB); - if (line != NULL) tfree(line); + if (line != NULL) taosMemoryFreeClear(line); taosCloseFile(&pFile); return 0; #endif @@ -606,7 +606,7 @@ int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int if (readIndex >= 4) break; } - if (line != NULL) tfree(line); + if (line != NULL) taosMemoryFreeClear(line); taosCloseFile(&pFile); if (readIndex < 4) { @@ -665,7 +665,7 @@ int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes) { *transmit_bytes = o_tbytes; } - if (line != NULL) tfree(line); + if (line != NULL) taosMemoryFreeClear(line); taosCloseFile(&pFile); return 0; diff --git a/source/util/src/talgo.c b/source/util/src/talgo.c index cb0a2d25a90a8999399e2e3bbfc3b4a4c838b0bd..48a0f327c4111d89dfd99437fc7c763d2d65b101 100644 --- a/source/util/src/talgo.c +++ b/source/util/src/talgo.c @@ -153,9 +153,9 @@ static void tqsortImpl(void *src, int32_t start, int32_t end, int64_t size, cons } void taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn) { - char *buf = calloc(1, size); // prepare the swap buffer + char *buf = taosMemoryCalloc(1, size); // prepare the swap buffer tqsortImpl(src, 0, (int32_t)numOfElem - 1, (int32_t)size, param, comparFn, buf); - tfree(buf); + taosMemoryFreeClear(buf); } void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size, __compar_fn_t compar, int32_t flags) { @@ -239,7 +239,7 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const child = 2 * parent + 1; if (swap == NULL) { - buf = calloc(1, size); + buf = taosMemoryCalloc(1, size); if (buf == NULL) { return; } @@ -288,7 +288,7 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const } if (swap == NULL) { - tfree(buf); + taosMemoryFreeClear(buf); } } } @@ -304,13 +304,13 @@ void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, } /* - char *buf = calloc(1, size); + char *buf = taosMemoryCalloc(1, size); for (i = len - 1; i > 0; i--) { doswap(elePtrAt(base, size, 0), elePtrAt(base, size, i)); taosheapadjust(base, size, 0, i - 1, parcompar, compar, parswap, swap, maxroot); } - tfree(buf); + taosMemoryFreeClear(buf); */ } diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 1127a212559ab832a6afdb367c8ab5d4b46a6877..a74b26a38666afe4e54817e9f8a773b156476e8b 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -23,15 +23,15 @@ SArray* taosArrayInit(size_t size, size_t elemSize) { size = TARRAY_MIN_SIZE; } - SArray* pArray = malloc(sizeof(SArray)); + SArray* pArray = taosMemoryMalloc(sizeof(SArray)); if (pArray == NULL) { return NULL; } pArray->size = 0; - pArray->pData = calloc(size, elemSize); + pArray->pData = taosMemoryCalloc(size, elemSize); if (pArray->pData == NULL) { - free(pArray); + taosMemoryFree(pArray); return NULL; } @@ -46,7 +46,7 @@ static int32_t taosArrayResize(SArray* pArray) { size_t size = pArray->capacity; size = (size << 1u); - void* tmp = realloc(pArray->pData, size * pArray->elemSize); + void* tmp = taosMemoryRealloc(pArray->pData, size * pArray->elemSize); if (tmp == NULL) { // reallocate failed, the original buffer remains return -1; } @@ -64,7 +64,7 @@ int32_t taosArrayEnsureCap(SArray* pArray, size_t newCap) { tsize = (tsize << 1u); } - pArray->pData = realloc(pArray->pData, tsize * pArray->elemSize); + pArray->pData = taosMemoryRealloc(pArray->pData, tsize * pArray->elemSize); if (pArray->pData == NULL) { return -1; } @@ -305,8 +305,8 @@ void taosArrayClear(SArray* pArray) { void* taosArrayDestroy(SArray* pArray) { if (pArray) { - free(pArray->pData); - free(pArray); + taosMemoryFree(pArray->pData); + taosMemoryFree(pArray); } return NULL; diff --git a/source/util/src/tbase64.c b/source/util/src/tbase64.c index c5119e8b2d43cce3e0cc0781c54206da8f9f0bea..a2f4ddbc51599ebc3909440b628d58819515fb2f 100644 --- a/source/util/src/tbase64.c +++ b/source/util/src/tbase64.c @@ -20,7 +20,7 @@ static char basis_64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz01 char *base64_encode(const uint8_t *value, int32_t vlen) { uint8_t oval = 0; - char *result = (char *)malloc((size_t)(vlen * 4) / 3 + 10); + char *result = (char *)taosMemoryMalloc((size_t)(vlen * 4) / 3 + 10); char *out = result; while (vlen >= 3) { *out++ = basis_64[value[0] >> 2]; @@ -53,7 +53,7 @@ static signed char index_64[128] = { uint8_t *base64_decode(const char *value, int32_t inlen, int32_t *outlen) { int32_t c1, c2, c3, c4; - uint8_t *result = (uint8_t *)malloc((size_t)(inlen * 3) / 4 + 1); + uint8_t *result = (uint8_t *)taosMemoryMalloc((size_t)(inlen * 3) / 4 + 1); uint8_t *out = result; *outlen = 0; @@ -93,7 +93,7 @@ uint8_t *base64_decode(const char *value, int32_t inlen, int32_t *outlen) { } base64_decode_error: - free(result); + taosMemoryFree(result); result = 0; *outlen = 0; diff --git a/source/util/src/tbuffer.c b/source/util/src/tbuffer.c index dc8c4b70c5107358a742e16ceef67584a111ff69..8552ccac2c66600ab851fb9770887f6e8a431c78 100644 --- a/source/util/src/tbuffer.c +++ b/source/util/src/tbuffer.c @@ -213,7 +213,7 @@ double tbufReadDouble(SBufferReader* buf) { // writer functions void tbufCloseWriter(SBufferWriter* buf) { - tfree(buf->data); + taosMemoryFreeClear(buf->data); // (*buf->allocator)( buf->data, 0 ); // potential memory leak. buf->data = NULL; buf->pos = 0; diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index cf5b62367867368ab1176dbf883ce2438defca51..c4e85a425e1c9af3beb8146ca58685756989d067 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -221,7 +221,7 @@ static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheNode * pCacheObj->freeFp(pNode->data); } - free(pNode); + taosMemoryFree(pNode); } static FORCE_INLINE STrashElem *doRemoveElemInTrashcan(SCacheObj *pCacheObj, STrashElem *pElem) { @@ -255,8 +255,8 @@ static FORCE_INLINE void doDestroyTrashcanElem(SCacheObj *pCacheObj, STrashElem pCacheObj->freeFp(pElem->pData->data); } - free(pElem->pData); - free(pElem); + taosMemoryFree(pElem->pData); + taosMemoryFree(pElem); } static void pushfrontNodeInEntryList(SCacheEntry *pEntry, SCacheNode *pNode) { @@ -358,7 +358,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext return NULL; } - SCacheObj *pCacheObj = (SCacheObj *)calloc(1, sizeof(SCacheObj)); + SCacheObj *pCacheObj = (SCacheObj *)taosMemoryCalloc(1, sizeof(SCacheObj)); if (pCacheObj == NULL) { uError("failed to allocate memory, reason:%s", strerror(errno)); return NULL; @@ -366,9 +366,9 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext // TODO add the auto extend procedure pCacheObj->capacity = 4096; - pCacheObj->pEntryList = calloc(pCacheObj->capacity, sizeof(SCacheEntry)); + pCacheObj->pEntryList = taosMemoryCalloc(pCacheObj->capacity, sizeof(SCacheEntry)); if (pCacheObj->pEntryList == NULL) { - free(pCacheObj); + taosMemoryFree(pCacheObj); uError("failed to allocate memory, reason:%s", strerror(errno)); return NULL; } @@ -381,8 +381,8 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext pCacheObj->extendLifespan = extendLifespan; // the TTL after the last access if (__trashcan_lock_init(pCacheObj) != 0) { - tfree(pCacheObj->pEntryList); - free(pCacheObj); + taosMemoryFreeClear(pCacheObj->pEntryList); + taosMemoryFree(pCacheObj); uError("failed to init lock, reason:%s", strerror(errno)); return NULL; @@ -432,7 +432,7 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v } atomic_sub_fetch_64(&pCacheObj->sizeInBytes, pNode->size); - tfree(pNode); + taosMemoryFreeClear(pNode); } else { taosAddToTrashcan(pCacheObj, pNode); uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, pNode->data); @@ -625,7 +625,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { pCacheObj->freeFp(pNode->data); } - free(pNode); + taosMemoryFree(pNode); } } @@ -703,7 +703,7 @@ void taosCacheCleanup(SCacheObj *pCacheObj) { SCacheNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, uint64_t duration) { size_t sizeInBytes = size + sizeof(SCacheNode) + keyLen; - SCacheNode *pNewNode = calloc(1, sizeInBytes); + SCacheNode *pNewNode = taosMemoryCalloc(1, sizeInBytes); if (pNewNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; uError("failed to allocate memory, reason:%s", strerror(errno)); @@ -735,7 +735,7 @@ void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheNode *pNode) { } __trashcan_wr_lock(pCacheObj); - STrashElem *pElem = calloc(1, sizeof(STrashElem)); + STrashElem *pElem = taosMemoryCalloc(1, sizeof(STrashElem)); pElem->pData = pNode; pElem->prev = NULL; pElem->next = NULL; @@ -802,9 +802,9 @@ void doCleanupDataCache(SCacheObj *pCacheObj) { __trashcan_lock_destroy(pCacheObj); - tfree(pCacheObj->pEntryList); - tfree(pCacheObj->name); - free(pCacheObj); + taosMemoryFreeClear(pCacheObj->pEntryList); + taosMemoryFreeClear(pCacheObj->name); + taosMemoryFree(pCacheObj); } static void doCacheRefresh(SCacheObj *pCacheObj, int64_t time, __cache_trav_fn_t fp, void *param1) { @@ -918,7 +918,7 @@ size_t taosCacheGetNumOfObj(const SCacheObj* pCacheObj) { SCacheIter* taosCacheCreateIter(const SCacheObj* pCacheObj) { ASSERT(pCacheObj != NULL); - SCacheIter* pIter = calloc(1, sizeof(SCacheIter)); + SCacheIter* pIter = taosMemoryCalloc(1, sizeof(SCacheIter)); pIter->pCacheObj = (SCacheObj*) pCacheObj; pIter->entryIndex = -1; pIter->index = -1; @@ -955,7 +955,7 @@ bool taosCacheIterNext(SCacheIter* pIter) { } if (pIter->numOfObj < pEntry->num) { - char *tmp = realloc(pIter->pCurrent, pEntry->num * POINTER_BYTES); + char *tmp = taosMemoryRealloc(pIter->pCurrent, pEntry->num * POINTER_BYTES); if (tmp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; taosRUnLockLatch(&pEntry->latch); @@ -1001,6 +1001,6 @@ void* taosCacheIterGetKey(const SCacheIter* pIter, size_t* len) { } void taosCacheDestroyIter(SCacheIter* pIter) { - tfree(pIter->pCurrent); - tfree(pIter); + taosMemoryFreeClear(pIter->pCurrent); + taosMemoryFreeClear(pIter); } \ No newline at end of file diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index ff7d2cf733c6af1043957d83d0dc24a8073db571..c98f6eb9be326317dd35dd678fed724ec30546ca 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -355,12 +355,12 @@ int32_t compareStrRegexCompNMatch(const void *pLeft, const void *pRight) { int32_t compareStrRegexComp(const void *pLeft, const void *pRight) { size_t sz = varDataLen(pRight); - char *pattern = malloc(sz + 1); + char *pattern = taosMemoryMalloc(sz + 1); memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); pattern[sz] = 0; sz = varDataLen(pLeft); - char *str = malloc(sz + 1); + char *str = taosMemoryMalloc(sz + 1); memcpy(str, varDataVal(pLeft), sz); str[sz] = 0; @@ -373,8 +373,8 @@ int32_t compareStrRegexComp(const void *pLeft, const void *pRight) { regerror(errCode, ®ex, msgbuf, sizeof(msgbuf)); uError("Failed to compile regex pattern %s. reason %s", pattern, msgbuf); regfree(®ex); - free(str); - free(pattern); + taosMemoryFree(str); + taosMemoryFree(pattern); return 1; } @@ -385,8 +385,8 @@ int32_t compareStrRegexComp(const void *pLeft, const void *pRight) { } int32_t result = (errCode == 0) ? 0 : 1; regfree(®ex); - free(str); - free(pattern); + taosMemoryFree(str); + taosMemoryFree(pattern); return result; } @@ -401,17 +401,17 @@ int32_t compareStrPatternMatch(const void *pLeft, const void *pRight) { SPatternCompareInfo pInfo = {'%', '_'}; assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN); - char *pattern = calloc(varDataLen(pRight) + 1, sizeof(char)); + char *pattern = taosMemoryCalloc(varDataLen(pRight) + 1, sizeof(char)); memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); size_t sz = varDataLen(pLeft); - char *buf = malloc(sz + 1); + char *buf = taosMemoryMalloc(sz + 1); memcpy(buf, varDataVal(pLeft), sz); buf[sz] = 0; int32_t ret = patternMatch(pattern, buf, sz, &pInfo); - free(buf); - free(pattern); + taosMemoryFree(buf); + taosMemoryFree(pattern); return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } @@ -424,11 +424,11 @@ int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight) { assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE); - char *pattern = calloc(varDataLen(pRight) + TSDB_NCHAR_SIZE, 1); + char *pattern = taosMemoryCalloc(varDataLen(pRight) + TSDB_NCHAR_SIZE, 1); memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); int32_t ret = WCSPatternMatch((TdUcs4*)pattern, (TdUcs4*)varDataVal(pLeft), varDataLen(pLeft) / TSDB_NCHAR_SIZE, &pInfo); - free(pattern); + taosMemoryFree(pattern); return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 404c9c8f7184b937fa28add3794868397a257085..96bb638b9555f0612032670b59e0f03a9444e2a8 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -29,7 +29,7 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url); int32_t cfgSetItem(SConfig *pConfig, const char *name, const char *value, ECfgSrcType stype); SConfig *cfgInit() { - SConfig *pCfg = calloc(1, sizeof(SConfig)); + SConfig *pCfg = taosMemoryCalloc(1, sizeof(SConfig)); if (pCfg == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -37,7 +37,7 @@ SConfig *cfgInit() { pCfg->array = taosArrayInit(32, sizeof(SConfigItem)); if (pCfg->array == NULL) { - free(pCfg); + taosMemoryFree(pCfg); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -75,7 +75,7 @@ int32_t cfgLoadFromArray(SConfig *pCfg, SArray *pArgs) { static void cfgFreeItem(SConfigItem *pItem) { if (pItem->dtype == CFG_DTYPE_STRING || pItem->dtype == CFG_DTYPE_DIR || pItem->dtype == CFG_DTYPE_LOCALE || pItem->dtype == CFG_DTYPE_CHARSET || pItem->dtype == CFG_DTYPE_TIMEZONE) { - tfree(pItem->str); + taosMemoryFreeClear(pItem->str); } if (pItem->array) { taosArrayDestroy(pItem->array); @@ -89,10 +89,10 @@ void cfgCleanup(SConfig *pCfg) { for (int32_t i = 0; i < size; ++i) { SConfigItem *pItem = taosArrayGet(pCfg->array, i); cfgFreeItem(pItem); - tfree(pItem->name); + taosMemoryFreeClear(pItem->name); } taosArrayDestroy(pCfg->array); - free(pCfg); + taosMemoryFree(pCfg); } } @@ -145,7 +145,7 @@ static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { return -1; } - tfree(pItem->str); + taosMemoryFreeClear(pItem->str); pItem->str = strdup(fullDir); if (pItem->str == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -224,7 +224,7 @@ static int32_t cfgSetString(SConfigItem *pItem, const char *value, ECfgSrcType s return -1; } - free(pItem->str); + taosMemoryFree(pItem->str); pItem->str = tmp; pItem->stype = stype; return 0; @@ -366,9 +366,9 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { if (taosArrayPush(pCfg->array, pItem) == NULL) { if (pItem->dtype == CFG_DTYPE_STRING) { - free(pItem->str); + taosMemoryFree(pItem->str); } - free(pItem->name); + taosMemoryFree(pItem->name); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -647,7 +647,7 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { } taosCloseFile(&pFile); - if (line != NULL) tfree(line); + if (line != NULL) taosMemoryFreeClear(line); uInfo("load from cfg file %s success", filepath); return 0; diff --git a/source/util/src/tcrc32c.c b/source/util/src/tcrc32c.c index f7e1060be0c42fa3704c9a471e5ae5719bdf90e2..66e9240cd59aec61acb97a3284d37fa10a79698e 100644 --- a/source/util/src/tcrc32c.c +++ b/source/util/src/tcrc32c.c @@ -19,13 +19,13 @@ */ #define _DEFAULT_SOURCE -#include "tcrc32c.h" -#include "tdef.h" - #if !defined(_TD_ARM_) && !defined(_TD_MIPS_) #include #endif +#include "tcrc32c.h" +#include "tdef.h" + #define POLY 0x82f63b78 #define LONG_SHIFT 8192 #define SHORT_SHIFT 256 diff --git a/source/util/src/tdes.c b/source/util/src/tdes.c index a7f5131c2641cf84dd665b7357ad6ed38cefb880..0e0193a12369321008221d0637852c0a909c6685 100644 --- a/source/util/src/tdes.c +++ b/source/util/src/tdes.c @@ -47,7 +47,7 @@ char* taosDesImp(uint8_t* key, char* src, uint32_t len, int32_t process_mode) { uint8_t processed_block[9] = {0}; key_set key_sets[17]; memset(key_sets, 0, sizeof(key_sets)); - char* dest = calloc(len + 1, 1); + char* dest = taosMemoryCalloc(len + 1, 1); generate_sub_keys(key, key_sets); for (uint32_t block_count = 0; block_count < number_of_blocks; block_count++) { @@ -68,12 +68,12 @@ char* taosDesEncode(int64_t key, char* src, int32_t len) { char* taosDesDecode(int64_t key, char* src, int32_t len) { uint8_t* keyStr = (uint8_t*)(&key); - char* temp = calloc(len + 8, 1); + char* temp = taosMemoryCalloc(len + 8, 1); memcpy(temp, src, len); len += 8; char* decode = taosDesImp(keyStr, temp, len, DECRYPTION_MODE); - free(temp); + taosMemoryFree(temp); return decode; } diff --git a/source/util/src/tencode.c b/source/util/src/tencode.c index 94b4cced46adb7f72e89b27b415a52ccf48a0253..c40d5b02e6fcce760e63d9a2c756c6bf352ee258 100644 --- a/source/util/src/tencode.c +++ b/source/util/src/tencode.c @@ -44,7 +44,7 @@ void tCoderClear(SCoder* pCoder) { pNode = TD_SLIST_HEAD(&(pCoder->stack)); if (pNode == NULL) break; TD_SLIST_POP(&(pCoder->stack)); - free(pNode); + taosMemoryFree(pNode); } } @@ -55,7 +55,7 @@ int32_t tStartEncode(SCoder* pCoder) { if (pCoder->data) { if (pCoder->size - pCoder->pos < sizeof(int32_t)) return -1; - pNode = malloc(sizeof(*pNode)); + pNode = taosMemoryMalloc(sizeof(*pNode)); if (pNode == NULL) return -1; pNode->data = pCoder->data; @@ -93,7 +93,7 @@ void tEndEncode(SCoder* pCoder) { TD_CODER_MOVE_POS(pCoder, len); - free(pNode); + taosMemoryFree(pNode); } } @@ -104,7 +104,7 @@ int32_t tStartDecode(SCoder* pCoder) { ASSERT(pCoder->type == TD_DECODER); if (tDecodeI32(pCoder, &len) < 0) return -1; - pNode = malloc(sizeof(*pNode)); + pNode = taosMemoryMalloc(sizeof(*pNode)); if (pNode == NULL) return -1; pNode->data = pCoder->data; @@ -133,5 +133,5 @@ void tEndDecode(SCoder* pCoder) { pCoder->pos = pCoder->size + pNode->pos; pCoder->size = pNode->size; - free(pNode); + taosMemoryFree(pNode); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index b7ab007cf55e36d0af45ffee8652264e7a105b18..005995c8e2aec90fde5db0d079f8c61a4f50ead0 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -270,6 +270,11 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CANT_PARALLEL, "Invalid stage to kill // mnode-topic TAOS_DEFINE_ERROR(TSDB_CODE_MND_UNSUPPORTED_TOPIC, "Topic with aggregation is unsupported") +// mnode-sma +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_ALREADY_EXIST, "SMA already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_NOT_EXIST, "SMA does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SMA_OPTION, "Invalid sma option") + // dnode TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_DND_OFFLINE, "Dnode is offline") @@ -305,6 +310,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, "Database write operat TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_SYNCING, "Database is syncing") TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_TSDB_STATE, "Invalid tsdb state") TAOS_DEFINE_ERROR(TSDB_CODE_VND_TB_NOT_EXIST, "Table not exists") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_SMA_NOT_EXIST, "SMA not exists") TAOS_DEFINE_ERROR(TSDB_CODE_VND_HASH_MISMATCH, "Hash value mismatch") // tsdb @@ -332,8 +338,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_MESSED_MSG, "TSDB messed message") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVLD_TAG_VAL, "TSDB invalid tag value") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_CACHE_LAST_ROW, "TSDB no cache last row data") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECREATED, "Table re-created") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_SMA_INDEX_IN_META, "No sma index in meta") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR, "TDB env open error") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_SMA_INDEX_IN_META, "No sma index in meta") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_SMA_STAT, "Invalid sma state") + // query TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, "Invalid handle") diff --git a/source/util/src/tfunctional.c b/source/util/src/tfunctional.c index c49fbdb50438effa0d980d50aaa00ea4a2d9428b..3b51d0046fa43222157eca76b509bf5696ef309c 100644 --- a/source/util/src/tfunctional.c +++ b/source/util/src/tfunctional.c @@ -17,21 +17,21 @@ #include "tfunctional.h" tGenericSavedFunc* genericSavedFuncInit(GenericVaFunc func, int32_t numOfArgs) { - tGenericSavedFunc* pSavedFunc = malloc(sizeof(tGenericSavedFunc) + numOfArgs * (sizeof(void*))); + tGenericSavedFunc* pSavedFunc = taosMemoryMalloc(sizeof(tGenericSavedFunc) + numOfArgs * (sizeof(void*))); if (pSavedFunc == NULL) return NULL; pSavedFunc->func = func; return pSavedFunc; } tI32SavedFunc* i32SavedFuncInit(I32VaFunc func, int32_t numOfArgs) { - tI32SavedFunc* pSavedFunc = malloc(sizeof(tI32SavedFunc) + numOfArgs * sizeof(void*)); + tI32SavedFunc* pSavedFunc = taosMemoryMalloc(sizeof(tI32SavedFunc) + numOfArgs * sizeof(void*)); if (pSavedFunc == NULL) return NULL; pSavedFunc->func = func; return pSavedFunc; } tVoidSavedFunc* voidSavedFuncInit(VoidVaFunc func, int32_t numOfArgs) { - tVoidSavedFunc* pSavedFunc = malloc(sizeof(tVoidSavedFunc) + numOfArgs * sizeof(void*)); + tVoidSavedFunc* pSavedFunc = taosMemoryMalloc(sizeof(tVoidSavedFunc) + numOfArgs * sizeof(void*)); if (pSavedFunc == NULL) return NULL; pSavedFunc->func = func; return pSavedFunc; diff --git a/source/util/src/thash.c b/source/util/src/thash.c index aac99bfe28b7c39cb7257f69bb362057f3f53fbb..88e5c12770a373fda703fd4e75c3a13b8f6606f6 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -33,7 +33,7 @@ #define FREE_HASH_NODE(_n) \ do { \ - tfree(_n); \ + taosMemoryFreeClear(_n); \ } while (0); struct SHashNode { @@ -238,7 +238,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp capacity = 4; } - SHashObj *pHashObj = (SHashObj *)calloc(1, sizeof(SHashObj)); + SHashObj *pHashObj = (SHashObj *)taosMemoryCalloc(1, sizeof(SHashObj)); if (pHashObj == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -254,26 +254,26 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp ASSERT((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); - pHashObj->hashList = (SHashEntry **)calloc(pHashObj->capacity, sizeof(void *)); + pHashObj->hashList = (SHashEntry **)taosMemoryCalloc(pHashObj->capacity, sizeof(void *)); if (pHashObj->hashList == NULL) { - free(pHashObj); + taosMemoryFree(pHashObj); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } pHashObj->pMemBlock = taosArrayInit(8, sizeof(void *)); if (pHashObj->pMemBlock == NULL) { - free(pHashObj->hashList); - free(pHashObj); + taosMemoryFree(pHashObj->hashList); + taosMemoryFree(pHashObj); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - void *p = calloc(pHashObj->capacity, sizeof(SHashEntry)); + void *p = taosMemoryCalloc(pHashObj->capacity, sizeof(SHashEntry)); if (p == NULL) { taosArrayDestroy(pHashObj->pMemBlock); - free(pHashObj->hashList); - free(pHashObj); + taosMemoryFree(pHashObj->hashList); + taosMemoryFree(pHashObj); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -440,14 +440,14 @@ void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void** if (size != NULL) { if (*d == NULL) { *size = pNode->dataLen; - *d = calloc(1, *size); + *d = taosMemoryCalloc(1, *size); if (*d == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } } else if (*size < pNode->dataLen) { *size = pNode->dataLen; - char* tmp = realloc(*d, *size); + char* tmp = taosMemoryRealloc(*d, *size); if (tmp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -567,24 +567,24 @@ void taosHashClear(SHashObj *pHashObj) { taosHashWUnlock(pHashObj); } -// the input paras should be SHashObj **, so the origin input will be set by tfree(*pHashObj) +// the input paras should be SHashObj **, so the origin input will be set by taosMemoryFreeClear(*pHashObj) void taosHashCleanup(SHashObj *pHashObj) { if (pHashObj == NULL) { return; } taosHashClear(pHashObj); - tfree(pHashObj->hashList); + taosMemoryFreeClear(pHashObj->hashList); // destroy mem block size_t memBlock = taosArrayGetSize(pHashObj->pMemBlock); for (int32_t i = 0; i < memBlock; ++i) { void *p = taosArrayGetP(pHashObj->pMemBlock, i); - tfree(p); + taosMemoryFreeClear(p); } taosArrayDestroy(pHashObj->pMemBlock); - free(pHashObj); + taosMemoryFree(pHashObj); } // for profile only @@ -623,7 +623,7 @@ void taosHashTableResize(SHashObj *pHashObj) { } int64_t st = taosGetTimestampUs(); - void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newCapacity); + void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity); if (pNewEntryList == NULL) { // uDebug("cache resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); return; @@ -632,7 +632,7 @@ void taosHashTableResize(SHashObj *pHashObj) { pHashObj->hashList = pNewEntryList; size_t inc = newCapacity - pHashObj->capacity; - void * p = calloc(inc, sizeof(SHashEntry)); + void * p = taosMemoryCalloc(inc, sizeof(SHashEntry)); for (int32_t i = 0; i < inc; ++i) { pHashObj->hashList[i + pHashObj->capacity] = (void *)((char *)p + i * sizeof(SHashEntry)); @@ -683,7 +683,7 @@ void taosHashTableResize(SHashObj *pHashObj) { } SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { - SHashNode *pNewNode = malloc(sizeof(SHashNode) + keyLen + dsize); + SHashNode *pNewNode = taosMemoryMalloc(sizeof(SHashNode) + keyLen + dsize); if (pNewNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/util/src/theap.c b/source/util/src/theap.c index 30af0483cc86c91b5ba2ba7a6fff81649f064a65..8c1a1db05709e3ef3eceb3329185d82f0b5485ed 100644 --- a/source/util/src/theap.c +++ b/source/util/src/theap.c @@ -19,7 +19,7 @@ size_t heapSize(Heap* heap) { return heap->nelts; } Heap* heapCreate(HeapCompareFn fn) { - Heap* heap = calloc(1, sizeof(Heap)); + Heap* heap = taosMemoryCalloc(1, sizeof(Heap)); if (heap == NULL) { return NULL; } @@ -30,7 +30,7 @@ Heap* heapCreate(HeapCompareFn fn) { return heap; } -void heapDestroy(Heap* heap) { free(heap); } +void heapDestroy(Heap* heap) { taosMemoryFree(heap); } HeapNode* heapMin(const Heap* heap) { return heap->min; } diff --git a/source/util/src/tidpool.c b/source/util/src/tidpool.c index 705cb0d2d3c7cd97a60c850843e30604a7995a08..b45113942c0fd22c5db95ea448af69780a317c04 100644 --- a/source/util/src/tidpool.c +++ b/source/util/src/tidpool.c @@ -18,12 +18,12 @@ #include "tlog.h" void *taosInitIdPool(int32_t maxId) { - id_pool_t *pIdPool = calloc(1, sizeof(id_pool_t)); + id_pool_t *pIdPool = taosMemoryCalloc(1, sizeof(id_pool_t)); if (pIdPool == NULL) return NULL; - pIdPool->freeList = calloc(maxId, sizeof(bool)); + pIdPool->freeList = taosMemoryCalloc(maxId, sizeof(bool)); if (pIdPool->freeList == NULL) { - free(pIdPool); + taosMemoryFree(pIdPool); return NULL; } @@ -79,13 +79,13 @@ void taosIdPoolCleanUp(id_pool_t *pIdPool) { uDebug("pool:%p is cleaned", pIdPool); - if (pIdPool->freeList) free(pIdPool->freeList); + if (pIdPool->freeList) taosMemoryFree(pIdPool->freeList); taosThreadMutexDestroy(&pIdPool->mutex); memset(pIdPool, 0, sizeof(id_pool_t)); - free(pIdPool); + taosMemoryFree(pIdPool); } int32_t taosIdPoolNumOfUsed(id_pool_t *pIdPool) { @@ -118,7 +118,7 @@ int32_t taosUpdateIdPool(id_pool_t *pIdPool, int32_t maxId) { return 0; } - bool *idList = calloc(maxId, sizeof(bool)); + bool *idList = taosMemoryCalloc(maxId, sizeof(bool)); if (idList == NULL) { return -1; } @@ -131,7 +131,7 @@ int32_t taosUpdateIdPool(id_pool_t *pIdPool, int32_t maxId) { bool *oldIdList = pIdPool->freeList; pIdPool->freeList = idList; - free(oldIdList); + taosMemoryFree(oldIdList); taosThreadMutexUnlock(&pIdPool->mutex); diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index a85da8cbbf56f6a69f582ecc1e34469786f7cc6d..0efcf517a9543730c01a86c19abced72eb3cf39a 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -255,7 +255,7 @@ int32_t tjsonMakeObject(const SJson* pJson, const char* pName, FToObject func, v if (NULL == pJsonObj) { return TSDB_CODE_FAILED; } - *pObj = calloc(1, objSize); + *pObj = taosMemoryCalloc(1, objSize); if (NULL == *pObj) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/util/src/tlist.c b/source/util/src/tlist.c index 5fccba614b5c65beb6893ad0be64a61732bb200d..1d17b4a9e17aa7cafdd89ba273770e8751f09066 100644 --- a/source/util/src/tlist.c +++ b/source/util/src/tlist.c @@ -22,7 +22,7 @@ void tdListInit(SList *list, int32_t eleSize) { } SList *tdListNew(int32_t eleSize) { - SList *list = (SList *)malloc(sizeof(SList)); + SList *list = (SList *)taosMemoryMalloc(sizeof(SList)); if (list == NULL) return NULL; tdListInit(list, eleSize); @@ -33,14 +33,14 @@ void tdListEmpty(SList *list) { SListNode *node; while ((node = TD_DLIST_HEAD(list)) != NULL) { TD_DLIST_POP(list, node); - free(node); + taosMemoryFree(node); } } void *tdListFree(SList *list) { if (list) { tdListEmpty(list); - free(list); + taosMemoryFree(list); } return NULL; @@ -51,7 +51,7 @@ void tdListPrependNode(SList *list, SListNode *node) { TD_DLIST_PREPEND(list, no void tdListAppendNode(SList *list, SListNode *node) { TD_DLIST_APPEND(list, node); } int32_t tdListPrepend(SList *list, void *data) { - SListNode *node = (SListNode *)malloc(sizeof(SListNode) + list->eleSize); + SListNode *node = (SListNode *)taosMemoryMalloc(sizeof(SListNode) + list->eleSize); if (node == NULL) return -1; memcpy((void *)(node->data), data, list->eleSize); @@ -61,7 +61,7 @@ int32_t tdListPrepend(SList *list, void *data) { } int32_t tdListAppend(SList *list, void *data) { - SListNode *node = (SListNode *)calloc(1, sizeof(SListNode) + list->eleSize); + SListNode *node = (SListNode *)taosMemoryCalloc(1, sizeof(SListNode) + list->eleSize); if (node == NULL) return -1; memcpy((void *)(node->data), data, list->eleSize); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 85db7883cf6b18ea05eb07be1faf90fc8672cc76..ef15f44f8fe5452f15dbe07f8e67228b98e45ddb 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -507,10 +507,10 @@ static void taosCloseLogByFd(TdFilePtr pFile) { static SLogBuff *taosLogBuffNew(int32_t bufSize) { SLogBuff *tLogBuff = NULL; - tLogBuff = calloc(1, sizeof(SLogBuff)); + tLogBuff = taosMemoryCalloc(1, sizeof(SLogBuff)); if (tLogBuff == NULL) return NULL; - LOG_BUF_BUFFER(tLogBuff) = malloc(bufSize); + LOG_BUF_BUFFER(tLogBuff) = taosMemoryMalloc(bufSize); if (LOG_BUF_BUFFER(tLogBuff) == NULL) goto _err; LOG_BUF_START(tLogBuff) = LOG_BUF_END(tLogBuff) = 0; @@ -524,8 +524,8 @@ static SLogBuff *taosLogBuffNew(int32_t bufSize) { return tLogBuff; _err: - tfree(LOG_BUF_BUFFER(tLogBuff)); - tfree(tLogBuff); + taosMemoryFreeClear(LOG_BUF_BUFFER(tLogBuff)); + taosMemoryFreeClear(tLogBuff); return NULL; } @@ -688,7 +688,7 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { int32_t compressSize = 163840; int32_t ret = 0; int32_t len = 0; - char *data = malloc(compressSize); + char *data = taosMemoryMalloc(compressSize); // gzFile dstFp = NULL; // srcFp = fopen(srcFileName, "r"); @@ -723,7 +723,7 @@ cmp_end: // if (dstFp) { // gzclose(dstFp); // } - free(data); + taosMemoryFree(data); return ret; } diff --git a/source/util/src/tlosertree.c b/source/util/src/tlosertree.c index 6349ab170c3df30abd94f8b978f16aa598bc22da..aeb9ce310b3a8802120fa58e52a3fc88a57b7eec 100644 --- a/source/util/src/tlosertree.c +++ b/source/util/src/tlosertree.c @@ -36,7 +36,7 @@ int32_t tMergeTreeCreate(SMultiwayMergeTreeInfo** pTree, uint32_t numOfSources, int32_t totalEntries = numOfSources << 1u; SMultiwayMergeTreeInfo* pTreeInfo = - (SMultiwayMergeTreeInfo*)calloc(1, sizeof(SMultiwayMergeTreeInfo) + sizeof(STreeNode) * totalEntries); + (SMultiwayMergeTreeInfo*)taosMemoryCalloc(1, sizeof(SMultiwayMergeTreeInfo) + sizeof(STreeNode) * totalEntries); if (pTreeInfo == NULL) { uError("allocate memory for loser-tree failed. reason:%s", strerror(errno)); return TAOS_SYSTEM_ERROR(errno); @@ -76,7 +76,7 @@ void tMergeTreeDestroy(SMultiwayMergeTreeInfo* pTree) { return; } - tfree(pTree); + taosMemoryFreeClear(pTree); } void tMergeTreeAdjust(SMultiwayMergeTreeInfo* pTree, int32_t idx) { diff --git a/source/util/src/tmallocator.c b/source/util/src/tmallocator.c index 057b908a583f297227982a8f101ab97a7ebe891a..0303af07e85306cac52a34746c2c80b9a16f649d 100644 --- a/source/util/src/tmallocator.c +++ b/source/util/src/tmallocator.c @@ -31,7 +31,7 @@ static size_t haUsage(SMemAllocator *pma); SMemAllocator *tdCreateHeapAllocator() { SMemAllocator *pma = NULL; - pma = calloc(1, sizeof(SMemAllocator) + sizeof(SHeapAllocator)); + pma = taosMemoryCalloc(1, sizeof(SMemAllocator) + sizeof(SHeapAllocator)); if (pma) { pma->impl = POINTER_SHIFT(pma, sizeof(SMemAllocator)); pma->malloc = haMalloc; @@ -53,7 +53,7 @@ static void *haMalloc(SMemAllocator *pma, size_t size) { size_t tsize = size + sizeof(size_t); SHeapAllocator *pha = (SHeapAllocator *)(pma->impl); - ptr = malloc(tsize); + ptr = taosMemoryMalloc(tsize); if (ptr) { *(size_t *)ptr = size; ptr = POINTER_SHIFT(ptr, sizeof(size_t)); @@ -97,7 +97,7 @@ static void haFree(SMemAllocator *pma, void *ptr) { /* TODO */ if (ptr) { size_t tsize = *(size_t *)POINTER_SHIFT(ptr, -sizeof(size_t)) + sizeof(size_t); atomic_fetch_sub_64(&(pha->tusage), tsize); - free(POINTER_SHIFT(ptr, -sizeof(size_t))); + taosMemoryFree(POINTER_SHIFT(ptr, -sizeof(size_t))); } } diff --git a/source/util/src/tmempool.c b/source/util/src/tmempool.c index d62e90397765af5c646a8fdfd3c07a04bfdd38ae..7bf8e94de9d434f140fc9c35fb0f521486b22367 100644 --- a/source/util/src/tmempool.c +++ b/source/util/src/tmempool.c @@ -37,7 +37,7 @@ mpool_h taosMemPoolInit(int32_t numOfBlock, int32_t blockSize) { return NULL; } - pool_p = (pool_t *)malloc(sizeof(pool_t)); + pool_p = (pool_t *)taosMemoryMalloc(sizeof(pool_t)); if (pool_p == NULL) { uError("mempool malloc failed\n"); return NULL; @@ -47,14 +47,14 @@ mpool_h taosMemPoolInit(int32_t numOfBlock, int32_t blockSize) { pool_p->blockSize = blockSize; pool_p->numOfBlock = numOfBlock; - pool_p->pool = (char *)malloc((size_t)(blockSize * numOfBlock)); - pool_p->freeList = (int32_t *)malloc(sizeof(int32_t) * (size_t)numOfBlock); + pool_p->pool = (char *)taosMemoryMalloc((size_t)(blockSize * numOfBlock)); + pool_p->freeList = (int32_t *)taosMemoryMalloc(sizeof(int32_t) * (size_t)numOfBlock); if (pool_p->pool == NULL || pool_p->freeList == NULL) { uError("failed to allocate memory\n"); - tfree(pool_p->freeList); - tfree(pool_p->pool); - tfree(pool_p); + taosMemoryFreeClear(pool_p->freeList); + taosMemoryFreeClear(pool_p->pool); + taosMemoryFreeClear(pool_p); return NULL; } @@ -120,8 +120,8 @@ void taosMemPoolCleanUp(mpool_h handle) { pool_t *pool_p = (pool_t *)handle; taosThreadMutexDestroy(&pool_p->mutex); - if (pool_p->pool) free(pool_p->pool); - if (pool_p->freeList) free(pool_p->freeList); + if (pool_p->pool) taosMemoryFree(pool_p->pool); + if (pool_p->freeList) taosMemoryFree(pool_p->freeList); memset(pool_p, 0, sizeof(*pool_p)); - free(pool_p); + taosMemoryFree(pool_p); } diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index a9d925ac076b3345e5abf99a6e598c112c2a251f..d834263b940ae6932bcd121b5a18791a491012d5 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -266,7 +266,7 @@ static SPageInfo* registerPage(SDiskbasedBuf* pBuf, int32_t groupId, int32_t pag pBuf->numOfPages += 1; - SPageInfo* ppi = malloc(sizeof(SPageInfo)); + SPageInfo* ppi = taosMemoryMalloc(sizeof(SPageInfo)); ppi->pageId = pageId; ppi->pData = NULL; @@ -330,7 +330,7 @@ static char* evacOneDataPage(SDiskbasedBuf* pBuf) { assert(d->pn == pn); d->pn = NULL; - tfree(pn); + taosMemoryFreeClear(pn); bufPage = flushPageToDisk(pBuf, d); } @@ -359,7 +359,7 @@ static SPageInfo* getPageInfoFromPayload(void* page) { int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, const char* id, const char* dir) { - *pBuf = calloc(1, sizeof(SDiskbasedBuf)); + *pBuf = taosMemoryCalloc(1, sizeof(SDiskbasedBuf)); SDiskbasedBuf* pPBuf = *pBuf; if (pPBuf == NULL) { @@ -386,7 +386,7 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem // init id hash table _hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT); pPBuf->groupSet = taosHashInit(10, fn, true, false); - pPBuf->assistBuf = malloc(pPBuf->pageSize + 2); // EXTRA BYTES + pPBuf->assistBuf = taosMemoryMalloc(pPBuf->pageSize + 2); // EXTRA BYTES pPBuf->all = taosHashInit(10, fn, true, false); char path[PATH_MAX] = {0}; @@ -422,7 +422,7 @@ void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t groupId, int32_t* pageId) { pi = *(SPageInfo**)pItem->data; pi->used = true; *pageId = pi->pageId; - tfree(pItem); + taosMemoryFreeClear(pItem); } else { // create a new pageinfo // register new id in this group *pageId = (++pBuf->allocateId); @@ -441,7 +441,7 @@ void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t groupId, int32_t* pageId) { // allocate buf if (availablePage == NULL) { - pi->pData = calloc(1, getAllocPageSize(pBuf->pageSize)); // add extract bytes in case of zipped buffer increased. + pi->pData = taosMemoryCalloc(1, getAllocPageSize(pBuf->pageSize)); // add extract bytes in case of zipped buffer increased. } else { pi->pData = availablePage; } @@ -483,7 +483,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { } if (availablePage == NULL) { - (*pi)->pData = calloc(1, getAllocPageSize(pBuf->pageSize)); + (*pi)->pData = taosMemoryCalloc(1, getAllocPageSize(pBuf->pageSize)); } else { (*pi)->pData = availablePage; } @@ -564,15 +564,15 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) { } taosRemoveFile(pBuf->path); - tfree(pBuf->path); + taosMemoryFreeClear(pBuf->path); SArray** p = taosHashIterate(pBuf->groupSet, NULL); while (p) { size_t n = taosArrayGetSize(*p); for (int32_t i = 0; i < n; ++i) { SPageInfo* pi = taosArrayGetP(*p, i); - tfree(pi->pData); - tfree(pi); + taosMemoryFreeClear(pi->pData); + taosMemoryFreeClear(pi); } taosArrayDestroy(*p); @@ -588,9 +588,9 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) { taosHashCleanup(pBuf->groupSet); taosHashCleanup(pBuf->all); - tfree(pBuf->id); - tfree(pBuf->assistBuf); - tfree(pBuf); + taosMemoryFreeClear(pBuf->id); + taosMemoryFreeClear(pBuf->assistBuf); + taosMemoryFreeClear(pBuf); } SPageInfo* getLastPageInfo(SIDList pList) { @@ -625,8 +625,8 @@ void dBufSetBufPageRecycled(SDiskbasedBuf* pBuf, void* pPage) { // add this pageinfo into the free page info list SListNode* pNode = tdListPopNode(pBuf->lruList, ppi->pn); - tfree(ppi->pData); - tfree(pNode); + taosMemoryFreeClear(ppi->pData); + taosMemoryFreeClear(pNode); tdListAppend(pBuf->freePgList, &ppi); } diff --git a/source/util/src/tprocess.c b/source/util/src/tprocess.c index f19a17fdb6dc26450801ac8ae1abe6391e355e45..9e55df44cfbeb225ad2b1a521414573cfe117dc3 100644 --- a/source/util/src/tprocess.c +++ b/source/util/src/tprocess.c @@ -163,13 +163,13 @@ static SProcQueue *taosProcQueueInit(int32_t size) { pQueue->bufferShmid = shmId; if (taosProcInitMutex(&pQueue->mutex, &pQueue->mutexShmid) != 0) { - free(pQueue); + taosMemoryFree(pQueue); return NULL; } if (tsem_init(&pQueue->sem, 1, 0) != 0) { taosProcDestroyMutex(pQueue->mutex, &pQueue->mutexShmid); - free(pQueue); + taosMemoryFree(pQueue); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -177,7 +177,7 @@ static SProcQueue *taosProcQueueInit(int32_t size) { if (taosProcInitMutex(&pQueue->mutex, &pQueue->mutexShmid) != 0) { taosProcDestroyMutex(pQueue->mutex, &pQueue->mutexShmid); tsem_destroy(&pQueue->sem); - free(pQueue); + taosMemoryFree(pQueue); return NULL; } @@ -195,7 +195,7 @@ static void taosProcQueueCleanup(SProcQueue *pQueue) { uDebug("proc:%s, queue:%p clean up", pQueue->name, pQueue); taosProcDestroyMutex(pQueue->mutex, &pQueue->mutexShmid); tsem_destroy(&pQueue->sem); - free(pQueue); + taosMemoryFree(pQueue); } } @@ -337,7 +337,7 @@ static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int32_t *pHea } SProcObj *taosProcInit(const SProcCfg *pCfg) { - SProcObj *pProc = calloc(1, sizeof(SProcObj)); + SProcObj *pProc = taosMemoryCalloc(1, sizeof(SProcObj)); if (pProc == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -350,7 +350,7 @@ SProcObj *taosProcInit(const SProcCfg *pCfg) { pProc->pParentQueue = taosProcQueueInit(pCfg->parentQueueSize); if (pProc->pChildQueue == NULL || pProc->pParentQueue == NULL) { taosProcQueueCleanup(pProc->pChildQueue); - free(pProc); + taosMemoryFree(pProc); return NULL; } @@ -447,7 +447,7 @@ void taosProcCleanup(SProcObj *pProc) { taosProcStop(pProc); taosProcQueueCleanup(pProc->pChildQueue); taosProcQueueCleanup(pProc->pParentQueue); - free(pProc); + taosMemoryFree(pProc); } } diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 70f2871f41b3228f58565a7ac9a7a3aa75b960fc..b01e1ea1daccd3679dcb5117219ec153c9716182 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -57,7 +57,7 @@ typedef struct STaosQall { } STaosQall; STaosQueue *taosOpenQueue() { - STaosQueue *queue = calloc(1, sizeof(STaosQueue)); + STaosQueue *queue = taosMemoryCalloc(1, sizeof(STaosQueue)); if (queue == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -96,11 +96,11 @@ void taosCloseQueue(STaosQueue *queue) { while (pNode) { pTemp = pNode; pNode = pNode->next; - free(pTemp); + taosMemoryFree(pTemp); } taosThreadMutexDestroy(&queue->mutex); - free(queue); + taosMemoryFree(queue); uDebug("queue:%p is closed", queue); } @@ -126,7 +126,7 @@ int32_t taosQueueSize(STaosQueue *queue) { } void *taosAllocateQitem(int32_t size) { - STaosQnode *pNode = calloc(1, sizeof(STaosQnode) + size); + STaosQnode *pNode = taosMemoryCalloc(1, sizeof(STaosQnode) + size); if (pNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -143,7 +143,7 @@ void taosFreeQitem(void *pItem) { char *temp = pItem; temp -= sizeof(STaosQnode); uTrace("item:%p, node:%p is freed", pItem, temp); - free(temp); + taosMemoryFree(temp); } int32_t taosWriteQitem(STaosQueue *queue, void *pItem) { @@ -193,9 +193,9 @@ int32_t taosReadQitem(STaosQueue *queue, void **ppItem) { return code; } -STaosQall *taosAllocateQall() { return calloc(1, sizeof(STaosQall)); } +STaosQall *taosAllocateQall() { return taosMemoryCalloc(1, sizeof(STaosQall)); } -void taosFreeQall(STaosQall *qall) { free(qall); } +void taosFreeQall(STaosQall *qall) { taosMemoryFree(qall); } int32_t taosReadAllQitems(STaosQueue *queue, STaosQall *qall) { int32_t code = 0; @@ -248,7 +248,7 @@ int32_t taosGetQitem(STaosQall *qall, void **ppItem) { void taosResetQitems(STaosQall *qall) { qall->current = qall->start; } STaosQset *taosOpenQset() { - STaosQset *qset = calloc(sizeof(STaosQset), 1); + STaosQset *qset = taosMemoryCalloc(sizeof(STaosQset), 1); if (qset == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -277,7 +277,7 @@ void taosCloseQset(STaosQset *qset) { taosThreadMutexDestroy(&qset->mutex); tsem_destroy(&qset->sem); - free(qset); + taosMemoryFree(qset); uDebug("qset:%p is closed", qset); } diff --git a/source/util/src/tref.c b/source/util/src/tref.c index 86633fe07a9769867b1992e00611e072aae1cff0..2e4c33bc8758f995c10e8efef6569e9f30cb67ee 100644 --- a/source/util/src/tref.c +++ b/source/util/src/tref.c @@ -65,15 +65,15 @@ int32_t taosOpenRef(int32_t max, void (*fp)(void *)) { taosThreadOnce(&tsRefModuleInit, taosInitRefModule); - nodeList = calloc(sizeof(SRefNode *), (size_t)max); + nodeList = taosMemoryCalloc(sizeof(SRefNode *), (size_t)max); if (nodeList == NULL) { terrno = TSDB_CODE_REF_NO_MEMORY; return -1; } - lockedBy = calloc(sizeof(int64_t), (size_t)max); + lockedBy = taosMemoryCalloc(sizeof(int64_t), (size_t)max); if (lockedBy == NULL) { - free(nodeList); + taosMemoryFree(nodeList); terrno = TSDB_CODE_REF_NO_MEMORY; return -1; } @@ -102,8 +102,8 @@ int32_t taosOpenRef(int32_t max, void (*fp)(void *)) { uTrace("rsetId:%d is opened, max:%d, fp:%p refSetNum:%d", rsetId, max, fp, tsRefSetNum); } else { rsetId = TSDB_CODE_REF_FULL; - free(nodeList); - free(lockedBy); + taosMemoryFree(nodeList); + taosMemoryFree(lockedBy); uTrace("run out of Ref ID, maximum:%d refSetNum:%d", TSDB_REF_OBJECTS, tsRefSetNum); } @@ -162,7 +162,7 @@ int64_t taosAddRef(int32_t rsetId, void *p) { return -1; } - pNode = calloc(sizeof(SRefNode), 1); + pNode = taosMemoryCalloc(sizeof(SRefNode), 1); if (pNode == NULL) { terrno = TSDB_CODE_REF_NO_MEMORY; return -1; @@ -445,7 +445,7 @@ static int32_t taosDecRefCount(int32_t rsetId, int64_t rid, int32_t remove) { uTrace("rsetId:%d p:%p rid:%" PRId64 " is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode); (*pSet->fp)(pNode->p); - free(pNode); + taosMemoryFree(pNode); taosDecRsetCount(pSet); } @@ -490,8 +490,8 @@ static void taosDecRsetCount(SRefSet *pSet) { pSet->max = 0; pSet->fp = NULL; - tfree(pSet->nodeList); - tfree(pSet->lockedBy); + taosMemoryFreeClear(pSet->nodeList); + taosMemoryFreeClear(pSet->lockedBy); tsRefSetNum--; uTrace("rsetId:%d is cleaned, refSetNum:%d count:%d", pSet->rsetId, tsRefSetNum, pSet->count); diff --git a/source/util/src/tsched.c b/source/util/src/tsched.c index 316c1f7ec138677e5125186760c6c747c804d8e2..2deba5077b37e227b507b4d982dd8d7883de1199 100644 --- a/source/util/src/tsched.c +++ b/source/util/src/tsched.c @@ -42,20 +42,20 @@ static void *taosProcessSchedQueue(void *param); static void taosDumpSchedulerStatus(void *qhandle, void *tmrId); void *taosInitScheduler(int32_t queueSize, int32_t numOfThreads, const char *label) { - SSchedQueue *pSched = (SSchedQueue *)calloc(sizeof(SSchedQueue), 1); + SSchedQueue *pSched = (SSchedQueue *)taosMemoryCalloc(sizeof(SSchedQueue), 1); if (pSched == NULL) { uError("%s: no enough memory for pSched", label); return NULL; } - pSched->queue = (SSchedMsg *)calloc(sizeof(SSchedMsg), queueSize); + pSched->queue = (SSchedMsg *)taosMemoryCalloc(sizeof(SSchedMsg), queueSize); if (pSched->queue == NULL) { uError("%s: no enough memory for queue", label); taosCleanUpScheduler(pSched); return NULL; } - pSched->qthread = calloc(sizeof(TdThread), numOfThreads); + pSched->qthread = taosMemoryCalloc(sizeof(TdThread), numOfThreads); if (pSched->qthread == NULL) { uError("%s: no enough memory for qthread", label); taosCleanUpScheduler(pSched); @@ -220,9 +220,9 @@ void taosCleanUpScheduler(void *param) { taosTmrStopA(&pSched->pTimer); } - if (pSched->queue) free(pSched->queue); - if (pSched->qthread) free(pSched->qthread); - free(pSched); // fix memory leak + if (pSched->queue) taosMemoryFree(pSched->queue); + if (pSched->qthread) taosMemoryFree(pSched->qthread); + taosMemoryFree(pSched); // fix memory leak } // for debug purpose, dump the scheduler status every 1min. diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index f7e56b6f3161aadff69d629dcbbfb305be94d49c..118fe58d2e9a58b3682aeb728b63e1d4f1dd77e1 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -28,7 +28,7 @@ static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **direction, SSkipListNode *pNode, bool isForward); static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, void *pData); static SSkipListNode *tSkipListNewNode(uint8_t level); -#define tSkipListFreeNode(n) tfree((n)) +#define tSkipListFreeNode(n) taosMemoryFreeClear((n)) static SSkipListNode *tSkipListPutImpl(SSkipList *pSkipList, void *pData, SSkipListNode **direction, bool isForward, bool hasDup); @@ -39,7 +39,7 @@ static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList); SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags, __sl_key_fn_t fn) { - SSkipList *pSkipList = (SSkipList *)calloc(1, sizeof(SSkipList)); + SSkipList *pSkipList = (SSkipList *)taosMemoryCalloc(1, sizeof(SSkipList)); if (pSkipList == NULL) return NULL; if (maxLevel > MAX_SKIP_LIST_LEVEL) { @@ -70,7 +70,7 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, _ } if (SL_IS_THREAD_SAFE(pSkipList)) { - pSkipList->lock = (TdThreadRwlock *)calloc(1, sizeof(TdThreadRwlock)); + pSkipList->lock = (TdThreadRwlock *)taosMemoryCalloc(1, sizeof(TdThreadRwlock)); if (pSkipList->lock == NULL) { tSkipListDestroy(pSkipList); return NULL; @@ -105,17 +105,17 @@ void tSkipListDestroy(SSkipList *pSkipList) { tSkipListFreeNode(pTemp); } - tfree(pSkipList->insertHandleFn); + taosMemoryFreeClear(pSkipList->insertHandleFn); tSkipListUnlock(pSkipList); if (pSkipList->lock != NULL) { taosThreadRwlockDestroy(pSkipList->lock); - tfree(pSkipList->lock); + taosMemoryFreeClear(pSkipList->lock); } tSkipListFreeNode(pSkipList->pHead); tSkipListFreeNode(pSkipList->pTail); - tfree(pSkipList); + taosMemoryFreeClear(pSkipList); } SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData) { @@ -345,7 +345,7 @@ void *tSkipListDestroyIter(SSkipListIterator *iter) { return NULL; } - tfree(iter); + taosMemoryFreeClear(iter); return NULL; } @@ -418,7 +418,7 @@ static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **direction, S } static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order) { - SSkipListIterator *iter = calloc(1, sizeof(SSkipListIterator)); + SSkipListIterator *iter = taosMemoryCalloc(1, sizeof(SSkipListIterator)); iter->pSkipList = pSkipList; iter->order = order; @@ -662,7 +662,7 @@ static int32_t initForwardBackwardPtr(SSkipList *pSkipList) { static SSkipListNode *tSkipListNewNode(uint8_t level) { int32_t tsize = sizeof(SSkipListNode) + sizeof(SSkipListNode *) * level * 2; - SSkipListNode *pNode = (SSkipListNode *)calloc(1, tsize); + SSkipListNode *pNode = (SSkipListNode *)taosMemoryCalloc(1, tsize); if (pNode == NULL) return NULL; pNode->level = level; diff --git a/source/util/src/tstrbuild.c b/source/util/src/tstrbuild.c index f191f69986a840c520703979f503406ab94ba9bc..2aae588046402e37569f5a2bde5ed5f72fa24346 100644 --- a/source/util/src/tstrbuild.c +++ b/source/util/src/tstrbuild.c @@ -20,7 +20,7 @@ void taosStringBuilderEnsureCapacity(SStringBuilder* sb, size_t size) { size += sb->pos; if (size > sb->size) { size *= 2; - void* tmp = realloc(sb->buf, size); + void* tmp = taosMemoryRealloc(sb->buf, size); if (tmp == NULL) { longjmp(sb->jb, 1); } @@ -39,7 +39,7 @@ char* taosStringBuilderGetResult(SStringBuilder* sb, size_t* len) { } void taosStringBuilderDestroy(SStringBuilder* sb) { - free(sb->buf); + taosMemoryFree(sb->buf); sb->buf = NULL; sb->pos = 0; sb->size = 0; diff --git a/source/util/src/tthread.c b/source/util/src/tthread.c index da76135b6899f4526d533b6d6faaca9e2f15261f..3a2247872972a49b6f84bae370fc4463c432c4d4 100644 --- a/source/util/src/tthread.c +++ b/source/util/src/tthread.c @@ -17,7 +17,7 @@ #include "tthread.h" TdThread* taosCreateThread(void* (*__start_routine)(void*), void* param) { - TdThread* pthread = (TdThread*)malloc(sizeof(TdThread)); + TdThread* pthread = (TdThread*)taosMemoryMalloc(sizeof(TdThread)); TdThreadAttr thattr; taosThreadAttrInit(&thattr); taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); @@ -25,7 +25,7 @@ TdThread* taosCreateThread(void* (*__start_routine)(void*), void* param) { taosThreadAttrDestroy(&thattr); if (ret != 0) { - free(pthread); + taosMemoryFree(pthread); return NULL; } return pthread; @@ -38,7 +38,7 @@ bool taosDestoryThread(TdThread* pthread) { taosThreadJoin(*pthread, NULL); } - free(pthread); + taosMemoryFree(pthread); return true; } diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index 5e59e61ea102742cef466ff2d9cf4112f557c242..fecc58c2367c90af8c94e98e029be9874749fd2a 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -141,7 +141,7 @@ static void timerAddRef(tmr_obj_t* timer) { atomic_add_fetch_8(&timer->refCount, static void timerDecRef(tmr_obj_t* timer) { if (atomic_sub_fetch_8(&timer->refCount, 1) == 0) { - free(timer); + taosMemoryFree(timer); } } @@ -351,7 +351,7 @@ tmr_h taosTmrStart(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* ha return NULL; } - tmr_obj_t* timer = (tmr_obj_t*)calloc(1, sizeof(tmr_obj_t)); + tmr_obj_t* timer = (tmr_obj_t*)taosMemoryCalloc(1, sizeof(tmr_obj_t)); if (timer == NULL) { tmrError("%s failed to allocated memory for new timer object.", ctrl->label); return NULL; @@ -513,7 +513,7 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* han } static void taosTmrModuleInit(void) { - tmrCtrls = malloc(sizeof(tmr_ctrl_t) * tsMaxTmrCtrl); + tmrCtrls = taosMemoryMalloc(sizeof(tmr_ctrl_t) * tsMaxTmrCtrl); if (tmrCtrls == NULL) { tmrError("failed to allocate memory for timer controllers."); return; @@ -539,7 +539,7 @@ static void taosTmrModuleInit(void) { } wheel->nextScanAt = now + wheel->resolution; wheel->index = 0; - wheel->slots = (tmr_obj_t**)calloc(wheel->size, sizeof(tmr_obj_t*)); + wheel->slots = (tmr_obj_t**)taosMemoryCalloc(wheel->size, sizeof(tmr_obj_t*)); if (wheel->slots == NULL) { tmrError("failed to allocate wheel slots"); return; @@ -548,7 +548,7 @@ static void taosTmrModuleInit(void) { } timerMap.count = 0; - timerMap.slots = (timer_list_t*)calloc(timerMap.size, sizeof(timer_list_t)); + timerMap.slots = (timer_list_t*)taosMemoryCalloc(timerMap.size, sizeof(timer_list_t)); if (timerMap.slots == NULL) { tmrError("failed to allocate hash map"); return; @@ -609,7 +609,7 @@ void taosTmrCleanUp(void* handle) { for (int32_t i = 0; i < tListLen(wheels); i++) { time_wheel_t* wheel = wheels + i; taosThreadMutexDestroy(&wheel->mutex); - free(wheel->slots); + taosMemoryFree(wheel->slots); } taosThreadMutexDestroy(&tmrCtrlMutex); @@ -619,12 +619,12 @@ void taosTmrCleanUp(void* handle) { tmr_obj_t* t = list->timers; while (t != NULL) { tmr_obj_t* next = t->mnext; - free(t); + taosMemoryFree(t); t = next; } } - free(timerMap.slots); - free(tmrCtrls); + taosMemoryFree(timerMap.slots); + taosMemoryFree(tmrCtrls); tmrCtrls = NULL; unusedTmrCtrl = NULL; diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index dc982596ada00621c583046c89e3ad0046f204be..8133e4d237f50c525ef653dd064854619e847dcc 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -147,7 +147,7 @@ char **strsplit(char *z, const char *delim, int32_t *num) { *num = 0; int32_t size = 4; - char **split = malloc(POINTER_BYTES * size); + char **split = taosMemoryMalloc(POINTER_BYTES * size); for (char *p = strsep(&z, delim); p != NULL; p = strsep(&z, delim)) { size_t len = strlen(p); @@ -158,7 +158,7 @@ char **strsplit(char *z, const char *delim, int32_t *num) { split[(*num)++] = p; if ((*num) >= size) { size = (size << 1); - split = realloc(split, POINTER_BYTES * size); + split = taosMemoryRealloc(split, POINTER_BYTES * size); assert(NULL != split); } } @@ -349,7 +349,7 @@ char *strbetween(char *string, char *begin, char *end) { char *_end = strstr(_begin + strlen(begin), end); int32_t size = (int32_t)(_end - _begin); if (_end != NULL && size > 0) { - result = (char *)calloc(1, size); + result = (char *)taosMemoryCalloc(1, size); memcpy(result, _begin + strlen(begin), size - +strlen(begin)); } } diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 044896d7a5824a2b4ddba4f76bdc0cecade286cd..992ec74b5b38a1009c7a3df7c66862bca810c4aa 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -22,7 +22,7 @@ typedef void *(*ThreadFp)(void *param); int32_t tQWorkerInit(SQWorkerPool *pool) { pool->qset = taosOpenQset(); - pool->workers = calloc(pool->max, sizeof(SQWorker)); + pool->workers = taosMemoryCalloc(pool->max, sizeof(SQWorker)); if (pool->workers == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -60,7 +60,7 @@ void tQWorkerCleanup(SQWorkerPool *pool) { } } - tfree(pool->workers); + taosMemoryFreeClear(pool->workers); taosCloseQset(pool->qset); taosThreadMutexDestroy(&pool->mutex); @@ -142,7 +142,7 @@ void tQWorkerFreeQueue(SQWorkerPool *pool, STaosQueue *queue) { int32_t tWWorkerInit(SWWorkerPool *pool) { pool->nextId = 0; - pool->workers = calloc(pool->max, sizeof(SWWorker)); + pool->workers = taosMemoryCalloc(pool->max, sizeof(SWWorker)); if (pool->workers == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -184,7 +184,7 @@ void tWWorkerCleanup(SWWorkerPool *pool) { } } - tfree(pool->workers); + taosMemoryFreeClear(pool->workers); taosThreadMutexDestroy(&pool->mutex); uInfo("worker:%s is closed", pool->name); diff --git a/source/util/test/hashTest.cpp b/source/util/test/hashTest.cpp index f4d2d9b47bf606ea2f86d1f0216a26df9445caf8..99f5a761c5d0d3a489176749883da981c847011d 100644 --- a/source/util/test/hashTest.cpp +++ b/source/util/test/hashTest.cpp @@ -158,7 +158,7 @@ void acquireRleaseTest() { const char *str2 = "aaaaaaa"; const char *str3 = "123456789"; - data.p = (char *)malloc(10); + data.p = (char *)taosMemoryMalloc(10); strcpy(data.p, str1); code = taosHashPut(hashTable, &key, sizeof(key), &data, sizeof(data)); @@ -177,7 +177,7 @@ void acquireRleaseTest() { strcpy(pdata->p, str3); - data.p = (char *)malloc(10); + data.p = (char *)taosMemoryMalloc(10); strcpy(data.p, str2); code = taosHashPut(hashTable, &key, sizeof(key), &data, sizeof(data)); ASSERT_EQ(code, 0); @@ -187,14 +187,14 @@ void acquireRleaseTest() { printf("%s,expect:%s", pdata->p, str3); ASSERT_TRUE(strcmp(pdata->p, str3) == 0); - tfree(pdata->p); + taosMemoryFreeClear(pdata->p); taosHashRelease(hashTable, pdata); num = taosHashGetSize(hashTable); ASSERT_EQ(num, 1); taosHashCleanup(hashTable); - tfree(data.p); + taosMemoryFreeClear(data.p); } } diff --git a/source/util/test/skiplistTest.cpp b/source/util/test/skiplistTest.cpp index 0b629f64aab9e07851af2d7b96e80121f4aaa2a4..7110b21aa9f6339c43a216a9c3c42393628d99b3 100644 --- a/source/util/test/skiplistTest.cpp +++ b/source/util/test/skiplistTest.cpp @@ -32,7 +32,7 @@ void doubleSkipListTest() { size = 0; // tSkipListNewNodeInfo(pSkipList, &level, &size); - // auto d = (SSkipListNode*)calloc(1, size + sizeof(double) * 2); + // auto d = (SSkipListNode*)taosMemoryCalloc(1, size + sizeof(double) * 2); // d->level = level; double key = 0.997; @@ -59,7 +59,7 @@ void doubleSkipListTest() { } if (size > 0) { - tfree(pNodes); + taosMemoryFreeClear(pNodes); } } @@ -83,7 +83,7 @@ void randKeyTest() { int32_t s = 0; tSkipListNewNodeInfo(pSkipList, &level, &s); - auto d = (SSkipListNode*)calloc(1, s + sizeof(int32_t) * 2); + auto d = (SSkipListNode*)taosMemoryCalloc(1, s + sizeof(int32_t) * 2); d->level = level; int32_t* key = (int32_t*)SL_GET_NODE_KEY(pSkipList, d); @@ -115,7 +115,7 @@ void stringKeySkiplistTest() { int32_t headsize = 0; tSkipListNewNodeInfo(pSkipList, &level, &headsize); - auto pNode = (SSkipListNode*)calloc(1, headsize + max_key_size + sizeof(double)); + auto pNode = (SSkipListNode*)taosMemoryCalloc(1, headsize + max_key_size + sizeof(double)); pNode->level = level; char* d = SL_GET_NODE_DATA(pNode); @@ -127,7 +127,7 @@ void stringKeySkiplistTest() { tSkipListNewNodeInfo(pSkipList, &level, &headsize); - pNode = (SSkipListNode*)calloc(1, headsize + max_key_size + sizeof(double)); + pNode = (SSkipListNode*)taosMemoryCalloc(1, headsize + max_key_size + sizeof(double)); pNode->level = level; d = SL_GET_NODE_DATA(pNode); @@ -153,7 +153,7 @@ void stringKeySkiplistTest() { tSkipListDestroy(pSkipList); - free(pRes); + taosMemoryFree(pRes); #endif tSkipListDestroy(pSkipList); @@ -167,7 +167,7 @@ void stringKeySkiplistTest() { int32_t n = sprintf(k, "abc_%d_%d", i, i); tSkipListNewNodeInfo(pSkipList, &level, &headsize); - auto pNode = (SSkipListNode*)calloc(1, headsize + 20 + sizeof(double)); + auto pNode = (SSkipListNode*)taosMemoryCalloc(1, headsize + 20 + sizeof(double)); pNode->level = level; char* d = SL_GET_NODE_DATA(pNode); @@ -197,7 +197,7 @@ void stringKeySkiplistTest() { tSkipListRemoveNode(pSkipList, pres[0]); if (num > 0) { - tfree(pres); + taosMemoryFreeClear(pres); } } @@ -219,7 +219,7 @@ void skiplistPerformanceTest() { int32_t unit = MAX_SKIP_LIST_LEVEL * POINTER_BYTES * 2 + sizeof(double) * 2 + sizeof(int16_t); - char* total = (char*)calloc(1, unit * size); + char* total = (char*)taosMemoryCalloc(1, unit * size); char* p = total; for (int32_t i = 0; i < size; ++i) { @@ -277,7 +277,7 @@ void skiplistPerformanceTest() { assert(SL_GET_SIZE(pSkipList) == size); tSkipListDestroy(pSkipList); - tfree(total); + taosMemoryFreeClear(total); } // todo not support duplicated key yet @@ -288,7 +288,7 @@ void duplicatedKeyTest() { for (int32_t j = 0; j < 5; ++j) { int32_t level, size; tSkipListNewNodeInfo(pSkipList, &level, &size); - SSkipListNode* d = (SSkipListNode*)calloc(1, size + sizeof(int32_t)); + SSkipListNode* d = (SSkipListNode*)taosMemoryCalloc(1, size + sizeof(int32_t)); d->level = level; int32_t* key = (int32_t*)SL_GET_NODE_KEY(pSkipList, d); key[0] = i; @@ -358,9 +358,9 @@ TEST(testCase, skiplist_test) { printf("-----%lf\n", pNodes[i]->key.dKey); } printf("the range query result size is: %d\n", size); - tfree(pNodes); + taosMemoryFreeClear(pNodes); - SSkipListKey *pKeys = malloc(sizeof(SSkipListKey) * 20); + SSkipListKey *pKeys = taosMemoryMalloc(sizeof(SSkipListKey) * 20); for (int32_t i = 0; i < 8; i += 2) { pKeys[i].dKey = i * 0.997; pKeys[i].nType = TSDB_DATA_TYPE_DOUBLE; @@ -372,9 +372,9 @@ TEST(testCase, skiplist_test) { for (int32_t i = 0; i < r; ++i) { // printf("%lf ", pNodes[i]->key.dKey); } - tfree(pNodes); + taosMemoryFreeClear(pNodes); - free(pKeys);*/ + taosMemoryFree(pKeys);*/ } #endif \ No newline at end of file diff --git a/source/util/test/stringTest.cpp b/source/util/test/stringTest.cpp index 95fba0cd3e2b62da2b2dadb0b7e592aef5893543..dee5fb30d30661d38413b2610a331b7c19bddd15 100644 --- a/source/util/test/stringTest.cpp +++ b/source/util/test/stringTest.cpp @@ -33,35 +33,35 @@ TEST(testCase, string_replace_test) { EXPECT_EQ(strlen(ret), 7); EXPECT_STREQ("7017027", ret); - free(ret); + taosMemoryFree(ret); char t4[] = "a01a02b03c04d05"; ret = strreplace(t4, "0", "9999999999"); EXPECT_EQ(strlen(ret), 5 * 10 + 10); EXPECT_STREQ("a99999999991a99999999992b99999999993c99999999994d99999999995", ret); - free(ret); + taosMemoryFree(ret); char t5[] = "abc"; ret = strreplace(t5, "abc", "12345678901234567890"); EXPECT_EQ(strlen(ret), 20); EXPECT_STREQ("12345678901234567890", ret); - free(ret); + taosMemoryFree(ret); char t6[] = "abc"; ret = strreplace(t6, "def", "abc"); EXPECT_EQ(strlen(ret), 3); EXPECT_STREQ("abc", ret); - free(ret); + taosMemoryFree(ret); char t7[] = "abcde000000000000001234"; ret = strreplace(t7, "ab", "0000000"); EXPECT_EQ(strlen(ret), 28); EXPECT_STREQ("0000000cde000000000000001234", ret); - free(ret); + taosMemoryFree(ret); char t8[] = "abc\ndef"; char t[] = {10, 0}; @@ -72,21 +72,21 @@ TEST(testCase, string_replace_test) { EXPECT_EQ(strlen(ret), 8); EXPECT_STREQ("abc\\ndef", ret); - free(ret); + taosMemoryFree(ret); char t9[] = "abc\\ndef"; ret = strreplace(t9, "\\n", "\n"); EXPECT_EQ(strlen(ret), 7); EXPECT_STREQ("abc\ndef", ret); - free(ret); + taosMemoryFree(ret); char t10[] = "abcdef"; ret = strreplace(t10, "", "0"); EXPECT_EQ(strlen(ret), 6); EXPECT_STREQ("abcdef", ret); - free(ret); + taosMemoryFree(ret); } #endif diff --git a/source/util/test/trefTest.c b/source/util/test/trefTest.c index a439a84562bd00675e1d50620cfdc30875c687aa..3174d57aef9b50583fc0b28d7bf389572bebdd0c 100644 --- a/source/util/test/trefTest.c +++ b/source/util/test/trefTest.c @@ -38,7 +38,7 @@ void *addRef(void *param) { printf("a"); id = random() % pSpace->refNum; if (pSpace->rid[id] <= 0) { - pSpace->p[id] = malloc(128); + pSpace->p[id] = taosMemoryMalloc(128); pSpace->rid[id] = taosAddRef(pSpace->rsetId, pSpace->p[id]); } taosUsleep(100); @@ -84,7 +84,7 @@ void *acquireRelease(void *param) { } void myfree(void *p) { - free(p); + taosMemoryFree(p); } void *openRefSpace(void *param) { @@ -98,8 +98,8 @@ void *openRefSpace(void *param) { return NULL; } - pSpace->p = (void **) calloc(sizeof(void *), pSpace->refNum); - pSpace->rid = calloc(pSpace->refNum, sizeof(int64_t)); + pSpace->p = (void **) taosMemoryCalloc(sizeof(void *), pSpace->refNum); + pSpace->rid = taosMemoryCalloc(pSpace->refNum, sizeof(int64_t)); TdThreadAttr thattr; taosThreadAttrInit(&thattr); @@ -121,7 +121,7 @@ void *openRefSpace(void *param) { taosCloseRef(pSpace->rsetId); uInfo("rsetId:%d main thread exit", pSpace->rsetId); - free(pSpace->p); + taosMemoryFree(pSpace->p); pSpace->p = NULL; return NULL; @@ -159,8 +159,8 @@ int main(int argc, char *argv[]) { taosInitLog("tref.log", 10); - SRefSpace *pSpaceList = (SRefSpace *) calloc(sizeof(SRefSpace), threads); - TdThread *pThreadList = (TdThread *) calloc(sizeof(TdThread), threads); + SRefSpace *pSpaceList = (SRefSpace *) taosMemoryCalloc(sizeof(SRefSpace), threads); + TdThread *pThreadList = (TdThread *) taosMemoryCalloc(sizeof(TdThread), threads); TdThreadAttr thattr; taosThreadAttrInit(&thattr); @@ -182,8 +182,8 @@ int main(int argc, char *argv[]) { int num = taosListRef(); printf("\nnumber of references:%d\n", num); - free(pSpaceList); - free(pThreadList); + taosMemoryFree(pSpaceList); + taosMemoryFree(pThreadList); taosCloseLog(); diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 7651e8708f134a2637340d9f53c496a101b055fb..de1050850d63a51010167f95f0de850c716732d8 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -227,13 +227,13 @@ int stmt_scol_func3(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -332,11 +332,11 @@ int stmt_scol_func3(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -357,13 +357,13 @@ int stmt_scol_func4(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1000 * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1000*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -455,11 +455,11 @@ int stmt_scol_func4(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -854,18 +854,18 @@ int stmt_funcb_autoctb1(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -1040,12 +1040,12 @@ int stmt_funcb_autoctb1(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -1067,18 +1067,18 @@ int stmt_funcb_autoctb2(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -1253,12 +1253,12 @@ int stmt_funcb_autoctb2(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -1281,18 +1281,18 @@ int stmt_funcb_autoctb3(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -1442,12 +1442,12 @@ int stmt_funcb_autoctb3(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -1472,18 +1472,18 @@ int stmt_funcb_autoctb4(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*5); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*5); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -1597,12 +1597,12 @@ int stmt_funcb_autoctb4(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -1625,18 +1625,18 @@ int stmt_funcb_autoctb_e1(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -1786,12 +1786,12 @@ int stmt_funcb_autoctb_e1(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -1814,18 +1814,18 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -2001,12 +2001,12 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -2031,18 +2031,18 @@ int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -2219,12 +2219,12 @@ int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -2246,18 +2246,18 @@ int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -2444,12 +2444,12 @@ int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -2473,18 +2473,18 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -2671,12 +2671,12 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -2697,13 +2697,13 @@ int stmt_funcb1(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -2830,11 +2830,11 @@ int stmt_funcb1(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -2854,13 +2854,13 @@ int stmt_funcb2(TAOS_STMT *stmt) { char bin[18000][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(18000 * sizeof(int)); + int *lb = taosMemoryMalloc(18000 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); - char* is_null = malloc(sizeof(char) * 18000); - char* no_null = malloc(sizeof(char) * 18000); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 18000); + char* no_null = taosMemoryMalloc(sizeof(char) * 18000); for (int i = 0; i < 18000; ++i) { lb[i] = 40; @@ -2988,11 +2988,11 @@ int stmt_funcb2(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -3012,13 +3012,13 @@ int stmt_funcb3(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -3151,11 +3151,11 @@ int stmt_funcb3(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -3177,13 +3177,13 @@ int stmt_funcb4(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -3310,11 +3310,11 @@ int stmt_funcb4(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -3336,13 +3336,13 @@ int stmt_funcb5(TAOS_STMT *stmt) { char bin[18000][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(18000 * sizeof(int)); + int *lb = taosMemoryMalloc(18000 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); - char* is_null = malloc(sizeof(char) * 18000); - char* no_null = malloc(sizeof(char) * 18000); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 18000); + char* no_null = taosMemoryMalloc(sizeof(char) * 18000); for (int i = 0; i < 18000; ++i) { lb[i] = 40; @@ -3463,11 +3463,11 @@ int stmt_funcb5(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -3480,12 +3480,12 @@ int stmt_funcb_ssz1(TAOS_STMT *stmt) { int b[30000]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 30000 * 3000); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 30000 * 3000); - int *lb = malloc(30000 * sizeof(int)); + int *lb = taosMemoryMalloc(30000 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); - char* no_null = malloc(sizeof(int) * 200000); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); + char* no_null = taosMemoryMalloc(sizeof(int) * 200000); for (int i = 0; i < 30000; ++i) { lb[i] = 40; @@ -3548,10 +3548,10 @@ int stmt_funcb_ssz1(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(no_null); return 0; } @@ -3571,13 +3571,13 @@ int stmt_funcb_s1(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -3705,11 +3705,11 @@ int stmt_funcb_s1(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -3733,13 +3733,13 @@ int stmt_funcb_sc1(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -3867,11 +3867,11 @@ int stmt_funcb_sc1(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -3891,13 +3891,13 @@ int stmt_funcb_sc2(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -4027,11 +4027,11 @@ int stmt_funcb_sc2(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -4051,13 +4051,13 @@ int stmt_funcb_sc3(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 60*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 60*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -4184,11 +4184,11 @@ int stmt_funcb_sc3(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -4247,7 +4247,7 @@ int sql_perf1(TAOS *taos) { TAOS_RES *result; for (int i = 0; i < 3000; i++) { - sql[i] = calloc(1, 1048576); + sql[i] = taosMemoryCalloc(1, 1048576); } int len = 0; @@ -4279,7 +4279,7 @@ int sql_perf1(TAOS *taos) { printf("insert total %d records, used %u seconds, avg:%.1f useconds\n", 3000*120*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*120*60)); for (int i = 0; i < 3000; i++) { - free(sql[i]); + taosMemoryFree(sql[i]); } return 0; @@ -4291,11 +4291,11 @@ int sql_perf1(TAOS *taos) { //one table 60 records one time int sql_perf_s1(TAOS *taos) { - char **sql = calloc(1, sizeof(char*) * 360000); + char **sql = taosMemoryCalloc(1, sizeof(char*) * 360000); TAOS_RES *result; for (int i = 0; i < 360000; i++) { - sql[i] = calloc(1, 9000); + sql[i] = taosMemoryCalloc(1, 9000); } int len = 0; @@ -4332,10 +4332,10 @@ int sql_perf_s1(TAOS *taos) { printf("insert total %d records, used %u seconds, avg:%.1f useconds\n", 3000*120*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*120*60)); for (int i = 0; i < 360000; i++) { - free(sql[i]); + taosMemoryFree(sql[i]); } - free(sql); + taosMemoryFree(sql); return 0; } @@ -4347,7 +4347,7 @@ int sql_s_perf1(TAOS *taos) { TAOS_RES *result; for (int i = 0; i < 3000; i++) { - sql[i] = calloc(1, 1048576); + sql[i] = taosMemoryCalloc(1, 1048576); } int len = 0; @@ -4379,7 +4379,7 @@ int sql_s_perf1(TAOS *taos) { printf("insert total %d records, used %u seconds, avg:%.1f useconds\n", 3000*120*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*120*60)); for (int i = 0; i < 3000; i++) { - free(sql[i]); + taosMemoryFree(sql[i]); } return 0; @@ -5066,7 +5066,7 @@ int main(int argc, char *argv[]) exit(1); } - TdThread *pThreadList = (TdThread *) calloc(sizeof(TdThread), 4); + TdThread *pThreadList = (TdThread *) taosMemoryCalloc(sizeof(TdThread), 4); TdThreadAttr thattr; taosThreadAttrInit(&thattr); diff --git a/tests/script/api/stmtBatchTest.c b/tests/script/api/stmtBatchTest.c index a6c20d282e44f48e8c54045fca9d377fca7d236a..b671b8bc068f0748e325714d12a23c4efa78e336 100644 --- a/tests/script/api/stmtBatchTest.c +++ b/tests/script/api/stmtBatchTest.c @@ -50,19 +50,19 @@ unsigned long long getCurrentTime(){ } static int stmt_bind_case_001(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -221,33 +221,33 @@ static int stmt_bind_case_001(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } static int stmt_bind_case_002(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -406,33 +406,33 @@ static int stmt_bind_case_002(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } static int stmt_bind_case_003(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -528,32 +528,32 @@ static int stmt_bind_case_003(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } static int stmt_bind_case_004(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -730,32 +730,32 @@ static int stmt_bind_case_004(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } static int stmt_bind_error_case_001(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -925,33 +925,33 @@ static int stmt_bind_error_case_001(TAOS_STMT *stmt, int tableNum, int rowsOfPer unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } static int stmt_bind_error_case_002(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -1110,32 +1110,32 @@ static int stmt_bind_error_case_002(TAOS_STMT *stmt, int tableNum, int rowsOfPer unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } static int stmt_bind_error_case_003(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -1314,14 +1314,14 @@ static int stmt_bind_error_case_003(TAOS_STMT *stmt, int tableNum, int rowsOfPer unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -2058,19 +2058,19 @@ static void runCase(TAOS *taos) { static int stmt_bind_case_001_long(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum, int64_t* startTs) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = *startTs; @@ -2231,14 +2231,14 @@ static int stmt_bind_case_001_long(TAOS_STMT *stmt, int tableNum, int rowsOfPerC unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -2325,19 +2325,19 @@ static void runCase_long(TAOS *taos) { test scene: insert into tb1 (ts,f1) values (?,?) */ static int stmt_specifyCol_bind_case_001(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -2455,14 +2455,14 @@ static int stmt_specifyCol_bind_case_001(TAOS_STMT *stmt, int tableNum, int rows unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -2472,19 +2472,19 @@ static int stmt_specifyCol_bind_case_001(TAOS_STMT *stmt, int tableNum, int rows test scene: insert into ? (ts,f1) values (?,?) */ static int stmt_specifyCol_bind_case_002(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -2602,14 +2602,14 @@ static int stmt_specifyCol_bind_case_002(TAOS_STMT *stmt, int tableNum, int rows unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -2619,19 +2619,19 @@ static int stmt_specifyCol_bind_case_002(TAOS_STMT *stmt, int tableNum, int rows test scene: insert into tb1 (ts,f1) values (?,?) */ static int stmt_specifyCol_bind_case_001_maxRows(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -2728,14 +2728,14 @@ static int stmt_specifyCol_bind_case_001_maxRows(TAOS_STMT *stmt, int tableNum, unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -3205,20 +3205,20 @@ static void SpecifyColumnBatchCase(TAOS *taos) { test scene: insert into tb1 (ts,f1) values (?,?) */ static int stmt_specifyCol_bind_case_001_autoCreateTbl(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int one_not_null = 0; int64_t tts = 1591060628000; @@ -3387,34 +3387,34 @@ static int stmt_specifyCol_bind_case_001_autoCreateTbl(TAOS_STMT *stmt, int tabl unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(tags); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(tags); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } static int stmt_specifyCol_bind_case_002_autoCreateTbl(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int one_not_null = 0; int64_t tts = 1591060628000; @@ -3583,35 +3583,35 @@ static int stmt_specifyCol_bind_case_002_autoCreateTbl(TAOS_STMT *stmt, int tabl unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(tags); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(tags); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } // some tags are null static int stmt_specifyCol_bind_case_003_autoCreateTbl(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int one_not_null = 0; int one_is_null = 1; @@ -3781,35 +3781,35 @@ static int stmt_specifyCol_bind_case_003_autoCreateTbl(TAOS_STMT *stmt, int tabl unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(tags); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(tags); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } // specify tags field, and not support , then is error case static int stmt_specifyCol_bind_case_004_autoCreateTbl(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int one_not_null = 0; int64_t tts = 1591060628000; @@ -3978,15 +3978,15 @@ static int stmt_specifyCol_bind_case_004_autoCreateTbl(TAOS_STMT *stmt, int tabl unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(tags); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(tags); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -4413,19 +4413,19 @@ char * taos_stmt_errstr(TAOS_STMT *stmt) 用于在其他stmt API 返回错误（ 3. 返回的错误码对于的错误消息； */ static int stmt_specifyCol_bind_error_case_001(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -4570,32 +4570,32 @@ static int stmt_specifyCol_bind_error_case_001(TAOS_STMT *stmt, int tableNum, in unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } static int stmt_specifyCol_bind_error_case_002(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -4715,14 +4715,14 @@ static int stmt_specifyCol_bind_error_case_002(TAOS_STMT *stmt, int tableNum, in unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -5079,8 +5079,8 @@ int main(int argc, char *argv[]) #if 0 printf("server:%s, threadNum:%d, rows:%d\n\n", serverIp, threadNum, g_rows); - TdThread *pThreadList = (TdThread *) calloc(sizeof(TdThread), (size_t)threadNum); - ThreadInfo* threadInfo = (ThreadInfo *) calloc(sizeof(ThreadInfo), (size_t)threadNum); + TdThread *pThreadList = (TdThread *) taosMemoryCalloc(sizeof(TdThread), (size_t)threadNum); + ThreadInfo* threadInfo = (ThreadInfo *) taosMemoryCalloc(sizeof(ThreadInfo), (size_t)threadNum); ThreadInfo* tInfo = threadInfo; for (int i = 0; i < threadNum; i++) { @@ -5105,8 +5105,8 @@ int main(int argc, char *argv[]) taosThreadJoin(pThreadList[i], NULL); } - free(pThreadList); - free(threadInfo); + taosMemoryFree(pThreadList); + taosMemoryFree(threadInfo); #endif taos = taos_connect(serverIp, "root", "taosdata", NULL, 0); diff --git a/tests/script/api/stmtTest.c b/tests/script/api/stmtTest.c index 46f1c7c8f8cbaf9523240e71458df23d78c54382..02d65de5677c96cea5256115bfb9a899dabe5f01 100644 --- a/tests/script/api/stmtTest.c +++ b/tests/script/api/stmtTest.c @@ -63,7 +63,7 @@ int main(int argc, char *argv[]) { execute_simple_sql(taos, "use test"); execute_simple_sql(taos, "create table super(ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(8), c6 smallint, c7 tinyint, c8 bool, c9 nchar(8), c10 timestamp) tags (t1 int, t2 bigint, t3 float, t4 double, t5 binary(8), t6 smallint, t7 tinyint, t8 bool, t9 nchar(8))"); - char *sql = calloc(1, 1024*1024); + char *sql = taosMemoryCalloc(1, 1024*1024); int sqlLen = 0; sqlLen = sprintf(sql, "create table"); for (int i = 0; i < 10; i++) { diff --git a/tests/script/api/stmt_function.c b/tests/script/api/stmt_function.c index a5427844d75a532e7c06936a39c84d90cf2cff0b..62f8fab29f1cf964bc9c9813a74191e080908e3e 100644 --- a/tests/script/api/stmt_function.c +++ b/tests/script/api/stmt_function.c @@ -50,7 +50,7 @@ void taos_stmt_init_test() { } void taos_stmt_preprare_test() { printf("start taos_stmt_prepare test\n"); - char *stmt_sql = calloc(1, 1048576); + char *stmt_sql = taosMemoryCalloc(1, 1048576); TAOS_STMT *stmt = NULL; assert(taos_stmt_prepare(stmt, stmt_sql, 0) != 0); void *taos = NULL; @@ -93,14 +93,14 @@ void taos_stmt_preprare_test() { assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); assert(taos_stmt_close(stmt) == 0); - free(stmt_sql); + taosMemoryFree(stmt_sql); printf("finish taos_stmt_prepare test\n"); } void taos_stmt_set_tbname_test() { printf("start taos_stmt_set_tbname test\n"); TAOS_STMT *stmt = NULL; - char *name = calloc(1, 200); + char *name = taosMemoryCalloc(1, 200); // ASM ERROR // assert(taos_stmt_set_tbname(stmt, name) != 0); void *taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); @@ -115,14 +115,14 @@ void taos_stmt_set_tbname_test() { stmt = taos_stmt_init(taos); assert(stmt != NULL); assert(taos_stmt_set_tbname(stmt, name) != 0); - char* stmt_sql = calloc(1, 1000); + char* stmt_sql = taosMemoryCalloc(1, 1000); sprintf(stmt_sql, "insert into ? values (?,?)"); assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); sprintf(name, "super"); assert(stmt != NULL); assert(taos_stmt_set_tbname(stmt, name) == 0); - free(name); - free(stmt_sql); + taosMemoryFree(name); + taosMemoryFree(stmt_sql); taos_stmt_close(stmt); printf("finish taos_stmt_set_tbname test\n"); } @@ -130,8 +130,8 @@ void taos_stmt_set_tbname_test() { void taos_stmt_set_tbname_tags_test() { printf("start taos_stmt_set_tbname_tags test\n"); TAOS_STMT *stmt = NULL; - char *name = calloc(1,20); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND)); + char *name = taosMemoryCalloc(1,20); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND)); // ASM ERROR // assert(taos_stmt_set_tbname_tags(stmt, name, tags) != 0); void *taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); @@ -146,7 +146,7 @@ void taos_stmt_set_tbname_tags_test() { execute_simple_sql(taos, "create table tb using super tags (1)"); stmt = taos_stmt_init(taos); assert(stmt != NULL); - char* stmt_sql = calloc(1, 1000); + char* stmt_sql = taosMemoryCalloc(1, 1000); sprintf(stmt_sql, "insert into ? using super tags (?) values (?,?)"); assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); assert(taos_stmt_set_tbname_tags(stmt, name, tags) != 0); @@ -159,9 +159,9 @@ void taos_stmt_set_tbname_tags_test() { tags->length = &tags->buffer_length; tags->is_null = NULL; assert(taos_stmt_set_tbname_tags(stmt, name, tags) == 0); - free(stmt_sql); - free(name); - free(tags); + taosMemoryFree(stmt_sql); + taosMemoryFree(name); + taosMemoryFree(tags); taos_stmt_close(stmt); printf("finish taos_stmt_set_tbname_tags test\n"); } @@ -169,7 +169,7 @@ void taos_stmt_set_tbname_tags_test() { void taos_stmt_set_sub_tbname_test() { printf("start taos_stmt_set_sub_tbname test\n"); TAOS_STMT *stmt = NULL; - char *name = calloc(1, 200); + char *name = taosMemoryCalloc(1, 200); // ASM ERROR // assert(taos_stmt_set_sub_tbname(stmt, name) != 0); void *taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); @@ -184,7 +184,7 @@ void taos_stmt_set_sub_tbname_test() { execute_simple_sql(taos, "create table tb using super tags (1)"); stmt = taos_stmt_init(taos); assert(stmt != NULL); - char* stmt_sql = calloc(1, 1000); + char* stmt_sql = taosMemoryCalloc(1, 1000); sprintf(stmt_sql, "insert into ? values (?,?)"); assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); assert(taos_stmt_set_sub_tbname(stmt, name) != 0); @@ -192,8 +192,8 @@ void taos_stmt_set_sub_tbname_test() { assert(taos_stmt_set_sub_tbname(stmt, name) == 0); // assert(taos_load_table_info(taos, "super, tb") == 0); // assert(taos_stmt_set_sub_tbname(stmt, name) == 0); - free(name); - free(stmt_sql); + taosMemoryFree(name); + taosMemoryFree(stmt_sql); assert(taos_stmt_close(stmt) == 0); printf("finish taos_stmt_set_sub_tbname test\n"); } @@ -213,12 +213,12 @@ void taos_stmt_bind_param_test() { execute_simple_sql(taos, "use stmt_test"); execute_simple_sql(taos, "create table super(ts timestamp, c1 int)"); stmt = taos_stmt_init(taos); - char* stmt_sql = calloc(1, 1000); + char* stmt_sql = taosMemoryCalloc(1, 1000); sprintf(stmt_sql, "insert into ? values (?,?)"); assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); assert(taos_stmt_bind_param(stmt, binds) != 0); - free(binds); - TAOS_BIND *params = calloc(2, sizeof(TAOS_BIND)); + taosMemoryFree(binds); + TAOS_BIND *params = taosMemoryCalloc(2, sizeof(TAOS_BIND)); int64_t ts = (int64_t)1591060628000; params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; params[0].buffer_length = sizeof(uint64_t); @@ -234,8 +234,8 @@ void taos_stmt_bind_param_test() { assert(taos_stmt_bind_param(stmt, params) != 0); assert(taos_stmt_set_tbname(stmt, "super") == 0); assert(taos_stmt_bind_param(stmt, params) == 0); - free(params); - free(stmt_sql); + taosMemoryFree(params); + taosMemoryFree(stmt_sql); taos_stmt_close(stmt); printf("finish taos_stmt_bind_param test\n"); } @@ -271,11 +271,11 @@ void taos_stmt_add_batch_test() { execute_simple_sql(taos, "create table super(ts timestamp, c1 int)"); stmt = taos_stmt_init(taos); assert(stmt != NULL); - char* stmt_sql = calloc(1, 1000); + char* stmt_sql = taosMemoryCalloc(1, 1000); sprintf(stmt_sql, "insert into ? values (?,?)"); assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); assert(taos_stmt_add_batch(stmt) != 0); - TAOS_BIND *params = calloc(2, sizeof(TAOS_BIND)); + TAOS_BIND *params = taosMemoryCalloc(2, sizeof(TAOS_BIND)); int64_t ts = (int64_t)1591060628000; params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; params[0].buffer_length = sizeof(uint64_t); @@ -291,8 +291,8 @@ void taos_stmt_add_batch_test() { assert(taos_stmt_set_tbname(stmt, "super") == 0); assert(taos_stmt_bind_param(stmt, params) == 0); assert(taos_stmt_add_batch(stmt) == 0); - free(params); - free(stmt_sql); + taosMemoryFree(params); + taosMemoryFree(stmt_sql); assert(taos_stmt_close(stmt) == 0); printf("finish taos_stmt_add_batch test\n"); } @@ -313,11 +313,11 @@ void taos_stmt_execute_test() { stmt = taos_stmt_init(taos); assert(stmt != NULL); assert(taos_stmt_execute(stmt) != 0); - char* stmt_sql = calloc(1, 1000); + char* stmt_sql = taosMemoryCalloc(1, 1000); sprintf(stmt_sql, "insert into ? values (?,?)"); assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); assert(taos_stmt_execute(stmt) != 0); - TAOS_BIND *params = calloc(2, sizeof(TAOS_BIND)); + TAOS_BIND *params = taosMemoryCalloc(2, sizeof(TAOS_BIND)); int64_t ts = (int64_t)1591060628000; params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; params[0].buffer_length = sizeof(uint64_t); @@ -336,8 +336,8 @@ void taos_stmt_execute_test() { assert(taos_stmt_execute(stmt) != 0); assert(taos_stmt_add_batch(stmt) == 0); assert(taos_stmt_execute(stmt) == 0); - free(params); - free(stmt_sql); + taosMemoryFree(params); + taosMemoryFree(stmt_sql); assert(taos_stmt_close(stmt) == 0); printf("finish taos_stmt_execute test\n"); } @@ -345,7 +345,7 @@ void taos_stmt_execute_test() { void taos_stmt_use_result_query(void *taos, char *col, int type) { TAOS_STMT *stmt = taos_stmt_init(taos); assert(stmt != NULL); - char *stmt_sql = calloc(1, 1024); + char *stmt_sql = taosMemoryCalloc(1, 1024); struct { int64_t c1; int32_t c2; @@ -372,7 +372,7 @@ void taos_stmt_use_result_query(void *taos, char *col, int type) { sprintf(stmt_sql, "select * from stmt_test.t1 where %s = ?", col); printf("stmt_sql: %s\n", stmt_sql); assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); - TAOS_BIND *params = calloc(1, sizeof(TAOS_BIND)); + TAOS_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_BIND)); params->buffer_type = type; params->is_null = NULL; switch(type){ @@ -434,8 +434,8 @@ void taos_stmt_use_result_query(void *taos, char *col, int type) { assert(result != NULL); print_result(result); assert(taos_stmt_close(stmt) == 0); - free(params); - free(stmt_sql); + taosMemoryFree(params); + taosMemoryFree(stmt_sql); taos_free_result(result); } diff --git a/tests/script/http/httpTest.c b/tests/script/http/httpTest.c index 0fe4265a9da316527d4d95239a2ec01d496b8a27..0298d5507dd7f54c3a5aa4ac95961f6a82f7c428 100644 --- a/tests/script/http/httpTest.c +++ b/tests/script/http/httpTest.c @@ -77,8 +77,8 @@ void execute(void *params) { char ip[] = "127.0.0.1"; int port = 6041; char page[] = "rest/sql"; - char *unique = calloc(1, 1024); - char *sql = calloc(1, 1024); + char *unique = taosMemoryCalloc(1, 1024); + char *sql = taosMemoryCalloc(1, 1024); ThreadObj *pThread = (ThreadObj *)params; printf("Thread %d started\n", pThread->threadId); sprintf(unique, "rest/sql/db%d",pThread->threadId); @@ -94,8 +94,8 @@ void execute(void *params) { sprintf(sql, "insert into t%d values (now + %ds, %d)", pThread->threadId, i, pThread->threadId); post(ip,port,unique, sql); } - free(unique); - free(sql); + taosMemoryFree(unique); + taosMemoryFree(sql); return; } @@ -103,7 +103,7 @@ void multiThread() { int numOfThreads = 100; int numOfTables = 100; int numOfRows = 1; - ThreadObj *threads = calloc((size_t)numOfThreads, sizeof(ThreadObj)); + ThreadObj *threads = taosMemoryCalloc((size_t)numOfThreads, sizeof(ThreadObj)); for (int i = 0; i < numOfThreads; i++) { ThreadObj *pthread = threads + i; TdThreadAttr thattr; @@ -117,7 +117,7 @@ void multiThread() { for (int i = 0; i < numOfThreads; i++) { taosThreadJoin(threads[i].pid, NULL); } - free(threads); + taosMemoryFree(threads); } int main() { diff --git a/tests/script/http/httpTestSqlUtc.c b/tests/script/http/httpTestSqlUtc.c index c6c5829b95d10794d65c6e910428c96800109b34..b93f13462395f5ed9035caf1f42969e6288964e1 100644 --- a/tests/script/http/httpTestSqlUtc.c +++ b/tests/script/http/httpTestSqlUtc.c @@ -77,8 +77,8 @@ void execute(void *params) { char ip[] = "127.0.0.1"; int port = 6041; char page[] = "rest/sqlutc"; - char *unique = calloc(1, 1024); - char *sql = calloc(1, 1024); + char *unique = taosMemoryCalloc(1, 1024); + char *sql = taosMemoryCalloc(1, 1024); ThreadObj *pThread = (ThreadObj *)params; printf("Thread %d started\n", pThread->threadId); sprintf(unique, "rest/sqlutc/db%d",pThread->threadId); @@ -94,8 +94,8 @@ void execute(void *params) { sprintf(sql, "insert into t%d values (now + %ds, %d)", pThread->threadId, i, pThread->threadId); post(ip,port,unique, sql); } - free(unique); - free(sql); + taosMemoryFree(unique); + taosMemoryFree(sql); return; } @@ -103,7 +103,7 @@ void multiThread() { int numOfThreads = 100; int numOfTables = 100; int numOfRows = 1; - ThreadObj *threads = calloc((size_t)numOfThreads, sizeof(ThreadObj)); + ThreadObj *threads = taosMemoryCalloc((size_t)numOfThreads, sizeof(ThreadObj)); for (int i = 0; i < numOfThreads; i++) { ThreadObj *pthread = threads + i; TdThreadAttr thattr; @@ -117,7 +117,7 @@ void multiThread() { for (int i = 0; i < numOfThreads; i++) { taosThreadJoin(threads[i].pid, NULL); } - free(threads); + taosMemoryFree(threads); } int main() { diff --git a/tests/script/http/httpTestSqlt.c b/tests/script/http/httpTestSqlt.c index 400428f471887da5f815a09aa875b2290f6398ba..d8d21ac137f836b2d83a5997c2eab3479416106a 100644 --- a/tests/script/http/httpTestSqlt.c +++ b/tests/script/http/httpTestSqlt.c @@ -77,8 +77,8 @@ void execute(void *params) { char ip[] = "127.0.0.1"; int port = 6041; char page[] = "rest/sqlt"; - char *unique = calloc(1, 1024); - char *sql = calloc(1, 1024); + char *unique = taosMemoryCalloc(1, 1024); + char *sql = taosMemoryCalloc(1, 1024); ThreadObj *pThread = (ThreadObj *)params; printf("Thread %d started\n", pThread->threadId); sprintf(unique, "rest/sqlt/db%d",pThread->threadId); @@ -94,8 +94,8 @@ void execute(void *params) { sprintf(sql, "insert into t%d values (now + %ds, %d)", pThread->threadId, i, pThread->threadId); post(ip,port,unique, sql); } - free(unique); - free(sql); + taosMemoryFree(unique); + taosMemoryFree(sql); return; } @@ -103,7 +103,7 @@ void multiThread() { int numOfThreads = 100; int numOfTables = 100; int numOfRows = 1; - ThreadObj *threads = calloc((size_t)numOfThreads, sizeof(ThreadObj)); + ThreadObj *threads = taosMemoryCalloc((size_t)numOfThreads, sizeof(ThreadObj)); for (int i = 0; i < numOfThreads; i++) { ThreadObj *pthread = threads + i; TdThreadAttr thattr; @@ -117,7 +117,7 @@ void multiThread() { for (int i = 0; i < numOfThreads; i++) { taosThreadJoin(threads[i].pid, NULL); } - free(threads); + taosMemoryFree(threads); } int main() { diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 821789de0aa1cc5322c505a8f3336bb2f702cedf..dd44baed27734b3574091b1f29d7d1c63e68203b 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -10,18 +10,26 @@ ./test.sh -f tsim/db/basic7.sim ./test.sh -f tsim/db/error1.sim -# ---- table -./test.sh -f tsim/table/basic1.sim - # ---- dnode ./test.sh -f tsim/dnode/basic1.sim # ---- insert -#./test.sh -f tsim/insert/basic0.sim +./test.sh -f tsim/insert/basic0.sim +./test.sh -f tsim/insert/basic1.sim +./test.sh -f tsim/insert/backquote.sim +./test.sh -f tsim/insert/null.sim # ---- query ./test.sh -f tsim/query/interval.sim +./test.sh -f tsim/query/interval-offset.sim + +# ---- show +./test.sh -f tsim/show/basic.sim + +# ---- table +./test.sh -f tsim/table/basic1.sim # ---- tmq ./test.sh -f tsim/tmq/basic.sim + #======================b1-end=============== diff --git a/tests/script/sh/sum_double.c b/tests/script/sh/sum_double.c index b4b0b81a0677b7de102dcfd228d09e53e3edda62..02979203617cf9da2b8e5b18bf326b56a45734d7 100644 --- a/tests/script/sh/sum_double.c +++ b/tests/script/sh/sum_double.c @@ -71,14 +71,14 @@ void sum_double_merge(char* data, int32_t numOfRows, char* dataOutput, int32_t* int sum_double_init(SUdfInit* buf) { buf->maybe_null=1; - buf->ptr = malloc(sizeof(int)); + buf->ptr = taosMemoryMalloc(sizeof(int)); printf("sum_double init\n"); return 0; } void sum_double_destroy(SUdfInit* buf) { - free(buf->ptr); + taosMemoryFree(buf->ptr); printf("sum_double destroy\n"); } diff --git a/tests/script/tsim/db/basic7.sim b/tests/script/tsim/db/basic7.sim index 27263ac606468bc188643d032c160fc80fce9d08..73651eed2f5dbf06c35d4b553d963bd610f6c97a 100644 --- a/tests/script/tsim/db/basic7.sim +++ b/tests/script/tsim/db/basic7.sim @@ -14,6 +14,7 @@ sql insert into tb1 values (now, 1); sql show stables if $rows != 1 then + print $rows return -1 endi diff --git a/tests/script/tsim/insert/backquote.sim b/tests/script/tsim/insert/backquote.sim index 59191fa2a5680f4de91fe2b9f4cebd4d3392a934..0c8b3df4afd20b85c9d44c743246fb60031fd295 100644 --- a/tests/script/tsim/insert/backquote.sim +++ b/tests/script/tsim/insert/backquote.sim @@ -36,9 +36,10 @@ while $dbCnt < 2 $dbCnt = $dbCnt + 1 print =============== create super table, include all type + print notes: after nchar show ok, modify binary to nchar sql create table `stable` (`timestamp` timestamp, `int` int, `binary` binary(16), `nchar` nchar(16)) tags (`float` float, `Binary` binary(16), `Nchar` nchar(16)) sql create table `Stable` (`timestamp` timestamp, `int` int, `Binary` binary(32), `Nchar` nchar(32)) tags (`float` float, `binary` binary(16), `nchar` nchar(16)) - + sql show stables print rows: $rows print $data00 $data01 @@ -47,10 +48,14 @@ while $dbCnt < 2 return -1 endi if $data00 != Stable then - return -1 + if $data00 != stable then + return -1 + endi endi - if $data10 != stable then - return -1 + if $data10 != Stable then + if $data10 != stable then + return -1 + endi endi print =============== create child table @@ -75,7 +80,7 @@ while $dbCnt < 2 sql insert into `Table` values(now+0s, 20, 'Table', 'Table')(now+1s, 21, 'Table', 'Table') sql insert into `TAble` values(now+0s, 30, 'TAble', 'TAble')(now+1s, 31, 'TAble', 'TAble') sql insert into `TABle` values(now+0s, 40, 'TABle', 'TABle')(now+4s, 41, 'TABle', 'TABle') - + print =============== query data sql select * from `table` print rows: $rows @@ -90,7 +95,8 @@ while $dbCnt < 2 if $data02 != table then return -1 endi - if $data03 != table then + if $data03 != table then + print expect table, actual $data03 return -1 endi @@ -145,26 +151,26 @@ while $dbCnt < 2 return -1 endi - print =============== query data from st, but not support select * from super table, waiting fix - sql select count(*) from `stable` - print rows: $rows - print $data00 $data01 $data02 $data03 - if $rows != 1 then - return -1 - endi - if $data00 != 4 then - return -1 - endi - sql select count(*) from `Stable` - print rows: $rows - print $data00 $data01 $data02 $data03 - if $rows != 1 then - return -1 - endi - if $data00 != 4 then - return -1 - endi - #sql select * from st + #print =============== query data from st, but not support select * from super table, waiting fix + #sql select count(*) from `stable` + #print rows: $rows + #print $data00 $data01 $data02 $data03 + #if $rows != 1 then + # return -1 + #endi + #if $data00 != 4 then + # return -1 + #endi + #sql select count(*) from `Stable` + #print rows: $rows + #print $data00 $data01 $data02 $data03 + #if $rows != 1 then + # return -1 + #endi + #if $data00 != 4 then + # return -1 + #endi + #sql select * from `stable` #if $rows != 4 then # return -1 #endi @@ -178,7 +184,7 @@ system sh/exec.sh -n dnode1 -s start $loop_cnt = 0 check_dnode_ready: $loop_cnt = $loop_cnt + 1 - sleep 100 + sleep 200 if $loop_cnt == 10 then print ====> dnode not ready! return -1 @@ -228,10 +234,14 @@ while $dbCnt < 2 return -1 endi if $data00 != Stable then - return -1 + if $data00 != stable then + return -1 + endi endi - if $data10 != stable then - return -1 + if $data10 != Stable then + if $data10 != stable then + return -1 + endi endi sql show tables @@ -313,26 +323,26 @@ while $dbCnt < 2 return -1 endi - print =============== query data from st, but not support select * from super table, waiting fix - sql select count(*) from `stable` - print rows: $rows - print $data00 $data01 $data02 $data03 - if $rows != 1 then - return -1 - endi - if $data00 != 4 then - return -1 - endi - sql select count(*) from `Stable` - print rows: $rows - print $data00 $data01 $data02 $data03 - if $rows != 1 then - return -1 - endi - if $data00 != 4 then - return -1 - endi - #sql select * from st + #print =============== query data from st, but not support select * from super table, waiting fix + #sql select count(*) from `stable` + #print rows: $rows + #print $data00 $data01 $data02 $data03 + #if $rows != 1 then + # return -1 + #endi + #if $data00 != 4 then + # return -1 + #endi + #sql select count(*) from `Stable` + #print rows: $rows + #print $data00 $data01 $data02 $data03 + #if $rows != 1 then + # return -1 + #endi + #if $data00 != 4 then + # return -1 + #endi + #sql select * from `stable` #if $rows != 4 then # return -1 #endi diff --git a/tests/script/tsim/insert/basic0.sim b/tests/script/tsim/insert/basic0.sim index 8592624cf3113fe46db5dbc668b45aaa256ce512..da74eb95e81f95ea0dd7fcffab3706632a37ff88 100644 --- a/tests/script/tsim/insert/basic0.sim +++ b/tests/script/tsim/insert/basic0.sim @@ -36,14 +36,14 @@ endi print =============== insert data, mode1: one row one table in sql print =============== insert data, mode1: mulit rows one table in sql -print =============== insert data, mode1: one rows mulit table in sql -print =============== insert data, mode1: mulit rows mulit table in sql +#print =============== insert data, mode1: one rows mulit table in sql +#print =============== insert data, mode1: mulit rows mulit table in sql sql insert into ct1 values(now+0s, 10, 2.0, 3.0) -sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, 12, 2.2, 3.2)(now+3s, 13, 2.3, 3.3) +sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) sql insert into ct2 values(now+0s, 10, 2.0, 3.0) -sql insert into ct2 values(now+1s, 11, 2.1, 3.1)(now+2s, 12, 2.2, 3.2)(now+3s, 13, 2.3, 3.3) -sql insert into ct1 values(now+4s, -14, -2.4, -3.4) ct2 values(now+4s, -14, -2.4, -3.4) -sql insert into ct1 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) ct2 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) +sql insert into ct2 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) +#sql insert into ct1 values(now+4s, -14, -2.4, -3.4) ct2 values(now+4s, -14, -2.4, -3.4) +#sql insert into ct1 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) ct2 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) sql insert into ct3 values('2021-01-01 00:00:00.000', 10, 2.0, 3.0) @@ -52,14 +52,11 @@ sql insert into ct3 values('2021-01-01 00:00:00.000', 10, 2.0, 3.0) print =============== query data from child table sql select * from ct1 print rows: $rows -print $data00 $data01 -print $data10 $data11 -print $data20 $data21 -print $data30 $data31 -print $data40 $data41 -print $data50 $data51 -print $data60 $data61 -if $rows != 7 then +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print $data30 $data31 $data32 $data33 +if $rows != 4 then return -1 endi if $data01 != 10 then @@ -84,48 +81,69 @@ endi print =============== select count(*) from child table sql select count(*) from ct1 +print rows: $rows +print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi - -print $data00 $data01 $data02 -if $data00 != 7 then +if $data00 != 4 then return -1 endi print =============== select count(column) from child table sql select count(ts), count(c1), count(c2), count(c3) from ct1 -print $data00 $data01 $data02 $data03 -if $data00 != 7 then +print rows: $rows +print $data00 $data01 $data02 $data03 +if $data00 != 4 then return -1 endi -if $data01 != 7 then +if $data01 != 4 then return -1 endi -if $data02 != 7 then +if $data02 != 4 then return -1 endi -if $data03 != 7 then +if $data03 != 4 then return -1 endi #print =============== select first(*)/first(column) from child table #sql select first(*) from ct1 -#sql select first(ts), first(c1), first(c2), first(c3) from ct1 +#print ====> select first(*) from ct1 +#print rows: $rows +#print $data00 $data01 $data02 $data03 + +sql select first(ts), first(c1), first(c2), first(c3) from ct1 +print ====> select first(ts), first(c1), first(c2), first(c3) from ct1 +print rows: $rows +print $data00 $data01 $data02 $data03 +if $rows != 1 then + return -1 +endi +if $data01 != 10 then + return -1 +endi +if $data02 != 2.00000 then + return -1 +endi +if $data03 != 3.000000000 then + return -1 +endi print =============== select min(column) from child table sql select min(c1), min(c2), min(c3) from ct1 -print $data00 $data01 $data02 $data03 +print rows: $rows +print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi -if $data00 != -16 then +if $data00 != -13 then return -1 endi -if $data01 != -2.60000 then +if $data01 != -2.30000 then return -1 endi -if $data02 != -3.600000000 then +if $data02 != -3.300000000 then return -1 endi @@ -135,13 +153,13 @@ print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi -if $data00 != 13 then +if $data00 != 11 then return -1 endi -if $data01 != 2.30000 then +if $data01 != 2.10000 then return -1 endi -if $data02 != 3.300000000 then +if $data02 != 3.100000000 then return -1 endi @@ -151,49 +169,53 @@ print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi -if $data00 != 1 then +if $data00 != -4 then return -1 endi -if $data01 != 1.099999905 then +if $data01 != -0.400000095 then return -1 endi -if $data02 != 2.100000000 then +if $data02 != -0.400000000 then return -1 endi -print =============== select column, from child table +print =============== select column without timestamp, from child table sql select c1, c2, c3 from ct1 +print rows: $rows print $data00 $data01 $data02 -#if $rows != 4 then -# return -1 -#endi -#if $data00 != 10 then -# return -1 -#endi -#if $data01 != 2.00000 then -# return -1 -#endi -#if $data02 != 3.000000000 then -# return -1 -#endi -#if $data10 != 11 then -# return -1 -#endi -#if $data11 != 2.10000 then -# return -1 -#endi -#if $data12 != 3.100000000 then -# return -1 -#endi -#if $data30 != 13 then -# return -1 -#endi -#if $data31 != 2.30000 then -# return -1 -#endi -#if $data32 != 3.300000000 then -# return -1 -#endi +print $data10 $data11 $data12 +print $data20 $data21 $data22 +print $data30 $data31 $data32 +if $rows != 4 then + return -1 +endi +if $data00 != 10 then + return -1 +endi +if $data01 != 2.00000 then + return -1 +endi +if $data02 != 3.000000000 then + return -1 +endi +if $data10 != 11 then + return -1 +endi +if $data11 != 2.10000 then + return -1 +endi +if $data12 != 3.100000000 then + return -1 +endi +if $data30 != -13 then + return -1 +endi +if $data31 != -2.30000 then + return -1 +endi +if $data32 != -3.300000000 then + return -1 +endi #=================================================================== #=================================================================== @@ -202,21 +224,53 @@ print $data00 $data01 $data02 #if $rows != 4 then # return -1 #endi + #print =============== select count(*) from supter table #sql select count(*) from stb +#print $data00 $data01 $data02 #if $rows != 1 then # return -1 #endi -# -#print $data00 $data01 $data02 -#if $data00 != 8 then +#if $data00 != 9 then # return -1 #endi -# + +print =============== select count(column) from supter table +sql select ts, c1, c2, c3 from stb +print rows: $rows +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print $data30 $data31 $data32 $data33 +print $data40 $data41 $data42 $data43 +print $data50 $data51 $data52 $data53 +print $data60 $data61 $data62 $data63 +print $data70 $data71 $data72 $data73 +print $data80 $data81 $data82 $data83 +if $rows != 9 then + return -1 +endi + +# The order of data from different sub tables in the super table is random, +# so this detection may fail randomly +if $data01 != 10 then + return -1 +endi +if $data02 != 2.00000 then + return -1 +endi +if $data03 != 3.000000000 then + return -1 +endi + #print =============== select count(column) from supter table #sql select count(ts), count(c1), count(c2), count(c3) from stb +#print rows: $rows #print $data00 $data01 $data02 $data03 -#if $data00 != 8 then +#print $data10 $data11 $data12 $data13 +#print $data20 $data21 $data22 $data23 +#print $data30 $data31 $data32 $data33 +#if $data00 != 9 then # return -1 #endi #if $data01 != 8 then @@ -229,7 +283,6 @@ print $data00 $data01 $data02 # return -1 #endi - #=================================================================== #=================================================================== @@ -237,9 +290,31 @@ print =============== stop and restart taosd, then again do query above system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start -sleep 2000 +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +print =============== query data from child table sql select * from ct1 -if $rows != 7 then +print rows: $rows +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print $data30 $data31 $data32 $data33 +if $rows != 4 then return -1 endi if $data01 != 10 then @@ -264,48 +339,69 @@ endi print =============== select count(*) from child table sql select count(*) from ct1 +print rows: $rows +print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi - -print $data00 $data01 $data02 -if $data00 != 7 then +if $data00 != 4 then return -1 endi print =============== select count(column) from child table sql select count(ts), count(c1), count(c2), count(c3) from ct1 -print $data00 $data01 $data02 $data03 -if $data00 != 7 then +print rows: $rows +print $data00 $data01 $data02 $data03 +if $data00 != 4 then return -1 endi -if $data01 != 7 then +if $data01 != 4 then return -1 endi -if $data02 != 7 then +if $data02 != 4 then return -1 endi -if $data03 != 7 then +if $data03 != 4 then return -1 endi #print =============== select first(*)/first(column) from child table #sql select first(*) from ct1 -#sql select first(ts), first(c1), first(c2), first(c3) from ct1 +#print ====> select first(*) from ct1 +#print rows: $rows +#print $data00 $data01 $data02 $data03 + +sql select first(ts), first(c1), first(c2), first(c3) from ct1 +print ====> select first(ts), first(c1), first(c2), first(c3) from ct1 +print rows: $rows +print $data00 $data01 $data02 $data03 +if $rows != 1 then + return -1 +endi +if $data01 != 10 then + return -1 +endi +if $data02 != 2.00000 then + return -1 +endi +if $data03 != 3.000000000 then + return -1 +endi print =============== select min(column) from child table sql select min(c1), min(c2), min(c3) from ct1 -print $data00 $data01 $data02 $data03 +print rows: $rows +print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi -if $data00 != -16 then +if $data00 != -13 then return -1 endi -if $data01 != -2.60000 then +if $data01 != -2.30000 then return -1 endi -if $data02 != -3.600000000 then +if $data02 != -3.300000000 then return -1 endi @@ -315,13 +411,13 @@ print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi -if $data00 != 13 then +if $data00 != 11 then return -1 endi -if $data01 != 2.30000 then +if $data01 != 2.10000 then return -1 endi -if $data02 != 3.300000000 then +if $data02 != 3.100000000 then return -1 endi @@ -331,14 +427,114 @@ print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi -if $data00 != 1 then +if $data00 != -4 then + return -1 +endi +if $data01 != -0.400000095 then + return -1 +endi +if $data02 != -0.400000000 then + return -1 +endi + +print =============== select column without timestamp, from child table +sql select c1, c2, c3 from ct1 +print rows: $rows +print $data00 $data01 $data02 +print $data10 $data11 $data12 +print $data20 $data21 $data22 +print $data30 $data31 $data32 +if $rows != 4 then + return -1 +endi +if $data00 != 10 then + return -1 +endi +if $data01 != 2.00000 then + return -1 +endi +if $data02 != 3.000000000 then + return -1 +endi +if $data10 != 11 then + return -1 +endi +if $data11 != 2.10000 then + return -1 +endi +if $data12 != 3.100000000 then + return -1 +endi +if $data30 != -13 then return -1 endi -if $data01 != 1.099999905 then +if $data31 != -2.30000 then return -1 endi -if $data02 != 2.100000000 then +if $data32 != -3.300000000 then return -1 endi +#=================================================================== +#=================================================================== + +#print =============== query data from stb +#sql select * from stb +#if $rows != 4 then +# return -1 +#endi + +#print =============== select count(*) from supter table +#sql select count(*) from stb +#print $data00 $data01 $data02 +#if $rows != 1 then +# return -1 +#endi +#if $data00 != 9 then +# return -1 +#endi + +print =============== select count(column) from supter table +sql select ts, c1, c2, c3 from stb +print rows: $rows +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print $data30 $data31 $data32 $data33 +print $data40 $data41 $data42 $data43 +print $data50 $data51 $data52 $data53 +print $data60 $data61 $data62 $data63 +print $data70 $data71 $data72 $data73 +print $data80 $data81 $data82 $data83 +if $rows != 9 then + return -1 +endi +# The order of data from different sub tables in the super table is random, +# so this detection may fail randomly +if $data01 != 10 then + return -1 +endi +if $data02 != 2.00000 then + return -1 +endi +if $data03 != 3.000000000 then + return -1 +endi + +#print =============== select count(column) from supter table +#sql select count(ts), count(c1), count(c2), count(c3) from stb +#print $data00 $data01 $data02 $data03 +#if $data00 != 8 then +# return -1 +#endi +#if $data01 != 8 then +# return -1 +#endi +#if $data02 != 8 then +# return -1 +#endi +#if $data03 != 8 then +# return -1 +#endi + #system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/insert/basic1.sim b/tests/script/tsim/insert/basic1.sim index 131044ac68e5269cd70393cd72c1cb0cbfd8ea3a..653a44a18a862be0924ffafeff86ec537a75604d 100644 --- a/tests/script/tsim/insert/basic1.sim +++ b/tests/script/tsim/insert/basic1.sim @@ -55,7 +55,8 @@ if $rows != 4 then return -1 endi -if $data01 != true then +if $data01 != 1 then + print expect 1, actual: $data01 return -1 endi @@ -80,7 +81,7 @@ system sh/exec.sh -n dnode1 -s start $loop_cnt = 0 check_dnode_ready: $loop_cnt = $loop_cnt + 1 - sleep 100 + sleep 200 if $loop_cnt == 10 then print ====> dnode not ready! return -1 @@ -105,7 +106,7 @@ if $rows != 4 then return -1 endi -if $data01 != true then +if $data01 != 1 then return -1 endi diff --git a/tests/script/tsim/insert/null.sim b/tests/script/tsim/insert/null.sim index 9dcc435486e8c8a87f19eaac2322870db58fd463..fbaef8cc94105b06c0b44b67ea23e57a2346c9cd 100644 --- a/tests/script/tsim/insert/null.sim +++ b/tests/script/tsim/insert/null.sim @@ -7,7 +7,7 @@ sql connect print =============== create database sql create database d0 sql show databases -if $rows != 1 then +if $rows != 2 then return -1 endi @@ -40,7 +40,7 @@ sql insert into ct1 values (now+7s, 14, NULL, 3.5, 95) sql insert into ct1 values (now+8s, 15, 2.5, NULL, 96) sql insert into ct1 values (now+9s, 16, 2.6, 3.6, NULL) sql insert into ct1 values (now+10s, NULL, NULL, NULL, NULL) -sql insert into ct1 values (now+11s, -2147483648, 2.7, 3.7, 97) +sql insert into ct1 values (now+11s, -2147483647, 2.7, 3.7, 97) #=================================================================== #=================================================================== @@ -75,7 +75,6 @@ endi # return -1 #endi - print =============== select count(*) from child table sql select count(*) from ct1 print ===> select count(*) from ct1 @@ -84,9 +83,7 @@ print ===> rows0: $data00 $data01 $data02 $data03 $data04 if $rows != 1 then return -1 endi - -print $data00 $data01 $data02 -if $data00 != 4 then +if $data00 != 12 then return -1 endi @@ -95,17 +92,16 @@ sql select count(ts), count(c1), count(c2), count(c3) from ct1 print ===> select count(ts), count(c1), count(c2), count(c3) from ct1 print ===> rows: $rows print ===> rows0: $data00 $data01 $data02 $data03 $data04 - -if $data00 != 4 then +if $data00 != 12 then return -1 endi -if $data01 != 4 then +if $data01 != 8 then return -1 endi -if $data02 != 4 then +if $data02 != 8 then return -1 endi -if $data03 != 4 then +if $data03 != 8 then return -1 endi @@ -121,7 +117,7 @@ print ===> rows0: $data00 $data01 $data02 $data03 $data04 if $rows != 1 then return -1 endi -if $data00 != 10 then +if $data00 != -2147483647 then return -1 endi if $data01 != 2.00000 then @@ -139,13 +135,13 @@ print ===> rows0: $data00 $data01 $data02 $data03 $data04 if $rows != 1 then return -1 endi -if $data00 != 13 then +if $data00 != 16 then return -1 endi -if $data01 != 2.30000 then +if $data01 != 2.70000 then return -1 endi -if $data02 != 3.300000000 then +if $data02 != 3.700000000 then return -1 endi @@ -157,13 +153,13 @@ print ===> rows0: $data00 $data01 $data02 $data03 $data04 if $rows != 1 then return -1 endi -if $data00 != 46 then +if $data00 != -2147483556 then return -1 endi -if $data01 != 8.599999905 then +if $data01 != 18.799999952 then return -1 endi -if $data02 != 12.600000000 then +if $data02 != 26.800000000 then return -1 endi @@ -172,42 +168,48 @@ sql select c1, c2, c3 from ct1 print ===> select c1, c2, c3 from ct1 print ===> rows: $rows print ===> rows0: $data00 $data01 $data02 $data03 $data04 -#if $rows != 4 then -# return -1 -#endi -#if $data00 != 10 then -# return -1 -#endi -#if $data01 != 2.00000 then -# return -1 -#endi -#if $data02 != 3.000000000 then -# return -1 -#endi -#if $data10 != 11 then -# return -1 -#endi -#if $data11 != 2.10000 then -# return -1 -#endi -#if $data12 != 3.100000000 then -# return -1 -#endi -#if $data30 != 13 then -# return -1 -#endi -#if $data31 != 2.30000 then -# return -1 -#endi -#if $data32 != 3.300000000 then -# return -1 -#endi +if $rows != 12 then + return -1 +endi +if $data00 != 10 then + return -1 +endi +if $data01 != 2.00000 then + return -1 +endi +if $data02 != 3.000000000 then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data11 != NULL then + return -1 +endi +if $data12 != NULL then + return -1 +endi +if $data30 != 11 then + return -1 +endi +if $data31 != NULL then + return -1 +endi +if $data32 != 3.200000000 then + return -1 +endi +if $data90 != 16 then + return -1 +endi +if $data91 != 2.60000 then + return -1 +endi +if $data92 != 3.600000000 then + return -1 +endi #=================================================================== #=================================================================== - -return - #print =============== query data from stb #sql select * from stb #print ===> @@ -218,19 +220,18 @@ return #endi #print =============== select count(*) from supter table #sql select count(*) from stb +#print $data00 $data01 $data02 #if $rows != 1 then # return -1 #endi -# -#print $data00 $data01 $data02 -#if $data00 != 8 then +#if $data00 != 12 then # return -1 #endi -# + #print =============== select count(column) from supter table #sql select count(ts), count(c1), count(c2), count(c3) from stb #print $data00 $data01 $data02 $data03 -#if $data00 != 8 then +#if $data00 != 12 then # return -1 #endi #if $data01 != 8 then @@ -243,7 +244,6 @@ return # return -1 #endi - #=================================================================== #=================================================================== @@ -251,9 +251,36 @@ print =============== stop and restart taosd, then again do query above system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start -sleep 2000 +print ===> waiting dnode ready +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +#=================================================================== +#=================================================================== +print =============== query data from child table sql select * from ct1 -if $rows != 4 then # after fix bug, modify 4 to 7 +print ===> select * from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 +if $rows != 12 then return -1 endi if $data01 != 10 then @@ -275,31 +302,33 @@ endi # return -1 #endi - print =============== select count(*) from child table sql select count(*) from ct1 +print ===> select count(*) from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 if $rows != 1 then return -1 endi - -print $data00 $data01 $data02 -if $data00 != 4 then +if $data00 != 12 then return -1 endi print =============== select count(column) from child table sql select count(ts), count(c1), count(c2), count(c3) from ct1 -print $data00 $data01 $data02 $data03 -if $data00 != 4 then +print ===> select count(ts), count(c1), count(c2), count(c3) from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +if $data00 != 12 then return -1 endi -if $data01 != 4 then +if $data01 != 8 then return -1 endi -if $data02 != 4 then +if $data02 != 8 then return -1 endi -if $data03 != 4 then +if $data03 != 8 then return -1 endi @@ -309,11 +338,13 @@ endi print =============== select min(column) from child table sql select min(c1), min(c2), min(c3) from ct1 -print $data00 $data01 $data02 $data03 +print ===> select min(c1), min(c2), min(c3) from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 if $rows != 1 then return -1 endi -if $data00 != 10 then +if $data00 != -2147483647 then return -1 endi if $data01 != 2.00000 then @@ -325,34 +356,119 @@ endi print =============== select max(column) from child table sql select max(c1), max(c2), max(c3) from ct1 -print $data00 $data01 $data02 $data03 +print ===> select max(c1), max(c2), max(c3) from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 if $rows != 1 then return -1 endi -if $data00 != 13 then +if $data00 != 16 then return -1 endi -if $data01 != 2.30000 then +if $data01 != 2.70000 then return -1 endi -if $data02 != 3.300000000 then +if $data02 != 3.700000000 then return -1 endi print =============== select sum(column) from child table sql select sum(c1), sum(c2), sum(c3) from ct1 -print $data00 $data01 $data02 $data03 +print ===> select sum(c1), sum(c2), sum(c3) from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 if $rows != 1 then return -1 endi -if $data00 != 46 then +if $data00 != -2147483556 then + return -1 +endi +if $data01 != 18.799999952 then + return -1 +endi +if $data02 != 26.800000000 then + return -1 +endi + +print =============== select column, from child table +sql select c1, c2, c3 from ct1 +print ===> select c1, c2, c3 from ct1 +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +if $rows != 12 then + return -1 +endi +if $data00 != 10 then + return -1 +endi +if $data01 != 2.00000 then + return -1 +endi +if $data02 != 3.000000000 then + return -1 +endi +if $data10 != NULL then + return -1 +endi +if $data11 != NULL then + return -1 +endi +if $data12 != NULL then + return -1 +endi +if $data30 != 11 then + return -1 +endi +if $data31 != NULL then return -1 endi -if $data01 != 8.599999905 then +if $data32 != 3.200000000 then return -1 endi -if $data02 != 12.600000000 then +if $data90 != 16 then return -1 endi +if $data91 != 2.60000 then + return -1 +endi +if $data92 != 3.600000000 then + return -1 +endi +#=================================================================== +#=================================================================== + +#print =============== query data from stb +#sql select * from stb +#print ===> +#print ===> rows: $rows +#print ===> rows0: $data00 $data01 $data02 $data03 $data04 +#if $rows != 4 then +# return -1 +#endi +#print =============== select count(*) from supter table +#sql select count(*) from stb +#print $data00 $data01 $data02 +#if $rows != 1 then +# return -1 +#endi +#if $data00 != 12 then +# return -1 +#endi + +#print =============== select count(column) from supter table +#sql select count(ts), count(c1), count(c2), count(c3) from stb +#print $data00 $data01 $data02 $data03 +#if $data00 != 12 then +# return -1 +#endi +#if $data01 != 8 then +# return -1 +#endi +#if $data02 != 8 then +# return -1 +#endi +#if $data03 != 8 then +# return -1 +#endi #system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/interval-offset.sim b/tests/script/tsim/query/interval-offset.sim index ffc9995ef519110f1b98d513da4da245c3122f0e..f188dff1ba91cb0311a58f992d00f3a4ef4430ae 100644 --- a/tests/script/tsim/query/interval-offset.sim +++ b/tests/script/tsim/query/interval-offset.sim @@ -5,8 +5,6 @@ sleep 500 sql connect print =============== create database -sql drop database d0 -x step1 -step1: sql create database d0 sql show databases if $rows != 2 then @@ -25,9 +23,11 @@ endi sql create table ct1 using stb tags ( 1 ) sql create table ct2 using stb tags ( 2 ) +sql create table ct3 using stb tags ( 3 ) +sql create table ct4 using stb tags ( 4 ) sql show tables print $rows $data00 $data10 $data20 -if $rows != 2 then +if $rows != 4 then return -1 endi @@ -88,13 +88,17 @@ print ===> rows4: $data40 $data41 $data42 $data43 $data44 print ===> rows5: $data50 $data51 $data52 $data53 $data54 print ===> rows6: $data60 $data61 $data62 $data63 $data64 print ===> rows7: $data70 $data71 $data72 $data73 $data74 -if $rows != 8 then +print ===> rows8: $data80 $data81 $data82 $data83 $data84 +if $rows != 9 then return -1 endi -if $data00 != 2 then +if $data00 != 1 then return -1 endi -if $data70 != 1 then +if $data70 != 2 then + return -1 +endi +if $data80 != 1 then return -1 endi @@ -128,238 +132,193 @@ endi sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h) sliding(12h) print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h) sliding(12h) print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 $data05 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 $data15 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 $data25 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 $data35 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 $data45 +print ===> rows5: $data50 $data51 $data52 $data53 $data54 $data55 +print ===> rows6: $data60 $data61 $data62 $data63 $data64 $data65 +print ===> rows7: $data70 $data71 $data72 $data73 $data74 $data75 +#if $rows != 8 then +# return -1 +#endi +#if $data00 != 1 then +# return -1 +#endi +#if $data10 != 2 then +# return -1 +#endi +#if $data20 != 2 then +# return -1 +#endi +#if $data30 != 2 then +# return -1 +#endi +#if $data40 != 2 then +# return -1 +#endi +#if $data50 != 2 then +# return -1 +#endi +#if $data60 != 2 then +# return -1 +#endi +#if $data70 != 1 then +# return -1 +#endi + +print =============== insert data into child table ct3 (n) +sql insert into ct3 values ( '2021-12-21 01:01:01.000', NULL ); +sql insert into ct3 values ( '2021-12-31 01:01:01.000', 1 ); +sql insert into ct3 values ( '2022-01-01 01:01:06.000', 2 ); +sql insert into ct3 values ( '2022-01-07 01:01:10.000', 3 ); +sql insert into ct3 values ( '2022-01-31 01:01:16.000', 4 ); +sql insert into ct3 values ( '2022-02-01 01:01:20.000', 5 ); +sql insert into ct3 values ( '2022-02-28 01:01:26.000', 6 ); +sql insert into ct3 values ( '2022-03-01 01:01:30.000', 7 ); +sql insert into ct3 values ( '2022-03-08 01:01:36.000', 8 ); + +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) +print ===> rows: $rows print ===> rows0: $data00 $data01 $data02 $data03 $data04 print ===> rows1: $data10 $data11 $data12 $data13 $data14 print ===> rows2: $data20 $data21 $data22 $data23 $data24 print ===> rows3: $data30 $data31 $data32 $data33 $data34 print ===> rows4: $data40 $data41 $data42 $data43 $data44 -print ===> rows5: $data50 $data51 $data52 $data53 $data54 -print ===> rows6: $data60 $data61 $data62 $data63 $data64 -print ===> rows7: $data70 $data71 $data72 $data73 $data74 -if $rows != 7 then - return -1 -endi -if $data00 != 2 then - return -1 -endi -if $data60 != 1 then - return -1 -endi - -return - - - - - - - - - -sql select count(*) from car interval(1n, 10d) order by ts desc -# tdSql.checkData(0, 1, 1) -# tdSql.checkData(1, 1, 2) -# tdSql.checkData(2, 1, 3) -# tdSql.checkData(3, 1, 3) -# tdSql.checkData(4, 1, 6) -# tdSql.checkData(5, 1, 1) -# tdSql.checkData(6, 1, 1) -# -sql select count(*) from car interval(2n, 5d) -# tdSql.checkData(0, 1, 1) -# tdSql.checkData(1, 1, 1) -# tdSql.checkData(2, 1, 6) -# tdSql.checkData(3, 1, 6) -# tdSql.checkData(4, 1, 3) - -sql select count(*) from car interval(2n) order by ts desc -# tdSql.checkData(0, 1, 3) -# tdSql.checkData(1, 1, 6) -# tdSql.checkData(2, 1, 6) -# tdSql.checkData(3, 1, 1) -# tdSql.checkData(4, 1, 1) -# -sql select count(*) from car interval(1y, 1n) -# tdSql.checkData(0, 1, 1) -# tdSql.checkData(1, 1, 8) -# tdSql.checkData(2, 1, 8) -# -sql select count(*) from car interval(1y, 2n) -# tdSql.checkData(0, 1, 1) -# tdSql.checkData(1, 1, 11) -# tdSql.checkData(2, 1, 5) - -sql select count(*) from car where ts > '2019-05-14 00:00:00' interval(1y, 5d) -# tdSql.checkData(0, 1, 6) -# tdSql.checkData(1, 1, 9) - - - - - - - - - - - - -sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol int) - -print ====== start create child tables and insert data -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - sql create table $tb using $mt tags( $i ) - - $x = 0 - while $x < $rowNum - $cc = $x * 60000 - $ms = 1601481600000 + $cc - - sql insert into $tb values ($ms , $x ) - $x = $x + 1 - endw - - $i = $i + 1 -endw - -print =============== step2 -$i = 1 -$tb = $tbPrefix . $i - -sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb interval(1m) -print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb interval(1m) -print ===> $rows $data01 $data05 -if $rows != $rowNum then - return -1 -endi -if $data00 != 1 then - return -1 -endi -if $data04 != 1 then - return -1 -endi - -#print =============== step3 -#$cc = 4 * 60000 -#$ms = 1601481600000 + $cc -#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms interval(1m) -#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms interval(1m) -#print ===> $rows $data01 $data05 -#if $rows != 5 then +#if $rows != 5 then # return -1 -#endi -#if $data00 != 1 then +#endi +#if $data00 != 1 then # return -1 -#endi -#if $data04 != 1 then +#endi +#if $data40 != 1 then # return -1 -#endi - -#print =============== step4 -#$cc = 40 * 60000 -#$ms = 1601481600000 + $cc +#endi -#$cc = 1 * 60000 -#$ms2 = 1601481600000 - $cc - -#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) -#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) -#print ===> $rows $data01 $data05 -#if $rows != 20 then +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) sliding(2w) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) sliding(2w) +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 +#if $rows != 5 then # return -1 -#endi -#if $data00 != 1 then +#endi +#if $data00 != 1 then # return -1 -#endi -#if $data04 != 1 then +#endi +#if $data40 != 1 then # return -1 -#endi - -#print =============== step5 -#$cc = 40 * 60000 -#$ms = 1601481600000 + $cc +#endi -#$cc = 1 * 60000 -#$ms2 = 1601481600000 - $cc - -#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) fill(value,0) -#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) fill(value,0) -#print ===> $rows $data21 $data25 -#if $rows != 42 then +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) sliding(4w) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) sliding(4w) +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 +print ===> rows5: $data50 $data51 $data52 $data53 $data54 +print ===> rows6: $data60 $data61 $data62 $data63 $data64 +print ===> rows7: $data70 $data71 $data72 $data73 $data74 +#if $rows != 8 then # return -1 -#endi -#if $data20 != 1 then +#endi +#if $data00 != 2 then # return -1 -#endi -#if $data24 != 1 then +#endi +#if $data70 != 1 then # return -1 -#endi - -#print =============== step6 -#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt interval(1m) -#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt interval(1m) -#print ===> $rows $data11 -#if $rows != 20 then +#endi + + +print =============== insert data into child table ct4 (y) +sql insert into ct4 values ( '2020-10-21 01:01:01.000', 1 ) +sql insert into ct4 values ( '2020-12-31 01:01:01.000', 2 ) +sql insert into ct4 values ( '2021-01-01 01:01:06.000', 3 ) +sql insert into ct4 values ( '2021-05-07 01:01:10.000', 4 ) +sql insert into ct4 values ( '2021-09-30 01:01:16.000', 5 ) +sql insert into ct4 values ( '2022-02-01 01:01:20.000', 6 ) +sql insert into ct4 values ( '2022-10-28 01:01:26.000', 7 ) +sql insert into ct4 values ( '2022-12-01 01:01:30.000', 8 ) +sql insert into ct4 values ( '2022-12-31 01:01:36.000', 9 ) + +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 +#if $rows != 5 then # return -1 -#endi -#if $data11 != 10 then +#endi +#if $data00 != 1 then # return -1 -#endi +#endi +#if $data40 != 1 then +# return -1 +#endi -#print =============== step7 -#$cc = 4 * 60000 -#$ms = 1601481600000 + $cc -#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms interval(1m) -#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms interval(1m) -#print ===> $rows $data11 -#if $rows != 5 then +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) sliding(6n) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) sliding(6n) +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 +#if $rows != 5 then # return -1 -#endi -#if $data11 != 10 then +#endi +#if $data00 != 1 then # return -1 -#endi - -#print =============== step8 -#$cc = 40 * 60000 -#$ms1 = 1601481600000 + $cc -# -#$cc = 1 * 60000 -#$ms2 = 1601481600000 - $cc -# -#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) -#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) -#print ===> $rows $data11 -#if $rows != 20 then +#endi +#if $data40 != 1 then # return -1 -#endi -#if $data11 != 10 then +#endi + +sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) sliding(12n) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) sliding(12n) +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 +print ===> rows5: $data50 $data51 $data52 $data53 $data54 +print ===> rows6: $data60 $data61 $data62 $data63 $data64 +print ===> rows7: $data70 $data71 $data72 $data73 $data74 +#if $rows != 8 then # return -1 -#endi -# -#print =============== step9 -#$cc = 40 * 60000 -#$ms1 = 1601481600000 + $cc -# -#$cc = 1 * 60000 -#$ms2 = 1601481600000 - $cc -# -#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) fill(value, 0) -#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) fill(value, 0) -#print ===> $rows $data11 -#if $rows != 42 then +#endi +#if $data00 != 2 then # return -1 -#endi -#if $data11 != 10 then +#endi +#if $data70 != 1 then # return -1 -#endi +#endi + + + + +#sql select count(*) from car interval(1n, 10d) order by ts desc +#sql select count(*) from car interval(2n, 5d) +#sql select count(*) from car interval(2n) order by ts desc +#sql select count(*) from car interval(1y, 1n) +#sql select count(*) from car interval(1y, 2n) +#sql select count(*) from car where ts > '2019-05-14 00:00:00' interval(1y, 5d) + + + + + -print =============== clear -#sql drop database $db -#sql show databases -#if $rows != 0 then -# return -1 -#endi #system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/show/basic.sim b/tests/script/tsim/show/basic.sim new file mode 100644 index 0000000000000000000000000000000000000000..cc926861422518d4cfb5fffcd5f2f9d273950f22 --- /dev/null +++ b/tests/script/tsim/show/basic.sim @@ -0,0 +1,215 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start + +# after mnode support, del sleep 2000, and check dnode status +sleep 2000 +sql connect + +#$loop_cnt = 0 +#check_dnode_ready: +# $loop_cnt = $loop_cnt + 1 +# sleep 200 +# if $loop_cnt == 10 then +# print ====> dnode not ready! +# return -1 +# endi +#sql show dnodes +#print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +#if $data00 != 1 then +# return -1 +#endi +#if $data04 != ready then +# goto check_dnode_ready +#endi +#sql connect + +# select */column from information_schema.xxxx; xxxx include: +# dnodes, mnodes, modules, qnodes, +# user_databases, user_functions, user_indexes, user_stables, user_streams, +# user_tables, user_table_distributed, user_users, vgroups, + +print =============== add dnode2 into cluster +sql create dnode $hostname port 7200 + +print =============== create database, stable, table +sql create database db vgroups 3 +sql use db +sql create table stb (ts timestamp, c int) tags (t int) +sql create table t0 using stb tags (0) + +print =============== run show xxxx +sql show dnodes +if $rows != 2 then + return -1 +endi + +sql show mnodes +if $rows != 1 then + return -1 +endi +#sql show modules +#sql show qnodes +sql show databases +if $rows != 2 then + return -1 +endi +#sql show functions + +#sql show indexes +sql show stables +if $rows != 1 then + return -1 +endi +#sql show streams, +sql show tables +if $rows != 1 then + return -1 +endi +#sql show user_table_distributed +sql show users +if $rows != 1 then + return -1 +endi +sql show vgroups +if $rows != 3 then + return -1 +endi + +print =============== run select * from information_schema.xxxx +sql select * from information_schema.`dnodes` +if $rows != 2 then + return -1 +endi +sql select * from information_schema.`mnodes` +if $rows != 1 then + return -1 +endi +#sql select * from information_schema.`modules` +#sql select * from information_schema.`qnodes` +sql select * from information_schema.user_databases +if $rows != 2 then + return -1 +endi +#sql select * from information_schema.user_functions +#sql select * from information_schema.user_indexes +sql select * from information_schema.user_stables +if $rows != 1 then + return -1 +endi +#sql select * from information_schema.user_streams +sql select * from information_schema.user_tables +if $rows != 1 then + return -1 +endi +#sql select * from information_schema.user_table_distributed +sql select * from information_schema.user_users +if $rows != 1 then + return -1 +endi +sql select * from information_schema.`vgroups` +if $rows != 3 then + return -1 +endi + +print ==== stop dnode1 and dnode2, and restart dnodes +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +print ==== again run show / select of above +print =============== run show xxxx +sql show dnodes +if $rows != 2 then + return -1 +endi + +sql show mnodes +if $rows != 1 then + return -1 +endi +#sql show modules +#sql show qnodes +sql show databases +if $rows != 2 then + return -1 +endi +#sql show functions + +#sql show indexes +sql show stables +if $rows != 1 then + return -1 +endi +#sql show streams, +sql show tables +if $rows != 1 then + return -1 +endi +#sql show user_table_distributed +sql show users +if $rows != 1 then + return -1 +endi +sql show vgroups +if $rows != 3 then + return -1 +endi + +print =============== run select * from information_schema.xxxx +sql select * from information_schema.`dnodes` +if $rows != 2 then + return -1 +endi +sql select * from information_schema.`mnodes` +if $rows != 1 then + return -1 +endi +#sql select * from information_schema.`modules` +#sql select * from information_schema.`qnodes` +sql select * from information_schema.user_databases +if $rows != 2 then + return -1 +endi +#sql select * from information_schema.user_functions +#sql select * from information_schema.user_indexes +sql select * from information_schema.user_stables +if $rows != 1 then + return -1 +endi +#sql select * from information_schema.user_streams +sql select * from information_schema.user_tables +if $rows != 1 then + return -1 +endi +#sql select * from information_schema.user_table_distributed +sql select * from information_schema.user_users +if $rows != 1 then + return -1 +endi +sql select * from information_schema.`vgroups` +if $rows != 3 then + return -1 +endi + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/table/basic1.sim b/tests/script/tsim/table/basic1.sim index a7f45ece083e244e1e573c46633b6938cd33b60e..bdb49cc9cdcd79739df3e547f6e98d8b052c6f0f 100644 --- a/tests/script/tsim/table/basic1.sim +++ b/tests/script/tsim/table/basic1.sim @@ -3,10 +3,33 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start sql connect +#=========== TD-14042 start +sql create database db1; +sql create database db2; +sql create database db3; + +sql use db1; +sql create table st1 (ts timestamp, i int) tags (j int); +sql create table tb1 using st1 tags(1); + +sql use db2; +sql create table st2 (ts timestamp, i int) tags (j int); +sql create table tb2 using st2 tags(1); + +sql use db3; +sql create table st3 (ts timestamp, i int) tags (j int); +sql create table tb3 using st3 tags(1); + +sql show tables +if $rows != 1 then + return -1 +endi +#=========== TD-14042 end + print =============== create database sql create database d1 sql show databases -if $rows != 2 then +if $rows != 5 then return -1 endi @@ -186,12 +209,32 @@ if $rows != 21 then return -1 endi -#system sh/exec.sh -n dnode1 -s stop -x SIGINT -#system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi -sleep 2000 print =============== query data sql select * from c1 +print rows: $rows +print $data00 $data01 +print $data10 $data11 +print $data20 $data21 if $rows != 3 then print $rows return -1 diff --git a/tests/script/tsim/testCaseSuite.sim b/tests/script/tsim/testCaseSuite.sim index 88f1911a2b6799873dd30faeacdb7557a3a155a9..4b700d2ca4f6f169457f72e9d819dab05c83337b 100644 --- a/tests/script/tsim/testCaseSuite.sim +++ b/tests/script/tsim/testCaseSuite.sim @@ -1,4 +1,6 @@ +run tsim/user/basic1.sim + run tsim/db/basic1.sim run tsim/db/basic6.sim run tsim/db/basic7.sim @@ -7,12 +9,16 @@ run tsim/db/error1.sim run tsim/dnode/basic1.sim run tsim/insert/basic0.sim -run tsim/insert/basic1.sim +run tsim/insert/basic1.sim +run tsim/insert/backquote.sim run tsim/insert/null.sim -run tsim/query/interval-offset.sim run tsim/query/interval.sim +#run tsim/query/interval-offset.sim # TD-14266 + +run tsim/show/basic.sim run tsim/table/basic1.sim -run tsim/user/basic1.sim +run tsim/tmq/basic.sim + diff --git a/tests/script/tsim/tmq/basic.sim b/tests/script/tsim/tmq/basic.sim index 31836f858092b07d472acff85533d0ba2f4dd71a..3e42c2cbd7880195111ea7a0add0ac0ed0dbd57b 100644 --- a/tests/script/tsim/tmq/basic.sim +++ b/tests/script/tsim/tmq/basic.sim @@ -47,9 +47,12 @@ sql drop database useless_db # -m startTimestamp, default is 1640966400000 [2022-01-01 00:00:00] # -g showMsgFlag, default is 0 # -#system_content ../../debug/tests/test/c/tmq_demo -c ../../sim/tsim/cfg -system ../../debug/tests/test/c/tmq_demo -c ../../sim/tsim/cfg -print result-> $system_content +print cmd===> system_content ../../debug/tests/test/c/tmq_demo -sim 1 -b 100 -c ../../sim/tsim/cfg -w ../../sim/dnode1/data/vnode/vnode4/wal +system_content ../../debug/tests/test/c/tmq_demo -sim 1 -b 100 -c ../../sim/tsim/cfg -w ../../sim/dnode1/data/vnode/vnode4/wal +print cmd result----> $system_content +if $system_content != @{consume success: 100}@ then + print not match in pos000 +endi sql show databases print ===> $rows $data00 $data01 $data02 $data03 @@ -78,4 +81,5 @@ endi if $data00 != 10000 then return -1 endi + #system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index 5e539780f46854b9ab85c0ba033c9b0923203a5e..4fca7d6245b6e5dcf18f4fa2103e6204ec4e6188 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -173,7 +173,7 @@ void showTables() { void *threadFunc(void *param) { SThreadInfo *pInfo = (SThreadInfo *)param; - char *qstr = malloc(batchNumOfTbl * batchNumOfRow * 128); + char *qstr = taosMemoryMalloc(batchNumOfTbl * batchNumOfRow * 128); int32_t code = 0; TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); @@ -284,7 +284,7 @@ void *threadFunc(void *param) { } taos_close(con); - free(qstr); + taosMemoryFree(qstr); return 0; } @@ -403,7 +403,7 @@ int32_t main(int32_t argc, char *argv[]) { TdThreadAttr thattr; taosThreadAttrInit(&thattr); taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - SThreadInfo *pInfo = (SThreadInfo *)calloc(numOfThreads, sizeof(SThreadInfo)); + SThreadInfo *pInfo = (SThreadInfo *)taosMemoryCalloc(numOfThreads, sizeof(SThreadInfo)); // int64_t numOfTablesPerThread = numOfTables / numOfThreads; // numOfTables = numOfTablesPerThread * numOfThreads; @@ -466,5 +466,5 @@ int32_t main(int32_t argc, char *argv[]) { } taosThreadAttrDestroy(&thattr); - free(pInfo); + taosMemoryFree(pInfo); } diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index 866bcb2ea0b9498915c11d33592f7c343a292ab8..205bc0a639859fb9789c415d304fb1ee24832aea 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -58,6 +58,7 @@ typedef struct { int32_t totalRowsOfPerTbl; int64_t startTimestamp; int32_t showMsgFlag; + int32_t simCase; int32_t totalRowsOfT2; } SConfInfo; @@ -66,7 +67,7 @@ static SConfInfo g_stConfInfo = { "tmqdb", "stb", "./tmqResult.txt", // output_file - "/data2/dnode/data/vnodes/vnode2/wal", + "/data2/dnode/data/vnode/vnode2/wal", 1, // threads 1, // tables 1, // vgroups @@ -77,6 +78,7 @@ static SConfInfo g_stConfInfo = { 10000, // total rows for per table 0, // 2020-01-01 00:00:00.000 0, // show consume msg switch + 0, // if run in sim case 10000, }; @@ -117,6 +119,8 @@ static void printHelp() { printf("%s%s%s%" PRId64 "\n", indent, indent, "startTimestamp, default is ", g_stConfInfo.startTimestamp); printf("%s%s\n", indent, "-g"); printf("%s%s%s%d\n", indent, indent, "showMsgFlag, default is ", g_stConfInfo.showMsgFlag); + printf("%s%s\n", indent, "-sim"); + printf("%s%s%s%d\n", indent, indent, "simCase, default is ", g_stConfInfo.simCase); exit(EXIT_SUCCESS); } @@ -160,14 +164,17 @@ void parseArgument(int32_t argc, char *argv[]) { g_stConfInfo.startTimestamp = atol(argv[++i]); } else if (strcmp(argv[i], "-g") == 0) { g_stConfInfo.showMsgFlag = atol(argv[++i]); + } else if (strcmp(argv[i], "-sim") == 0) { + g_stConfInfo.simCase = atol(argv[++i]); } else { - pPrint("%s unknow para: %s %s", GREEN, argv[++i], NC); + printf("%s unknow para: %s %s", GREEN, argv[++i], NC); exit(-1); } } g_stConfInfo.totalRowsOfT2 = g_stConfInfo.totalRowsOfPerTbl * g_stConfInfo.ratio; +#if 0 pPrint("%s configDir:%s %s", GREEN, configDir, NC); pPrint("%s dbName:%s %s", GREEN, g_stConfInfo.dbName, NC); pPrint("%s stbName:%s %s", GREEN, g_stConfInfo.stbName, NC); @@ -184,6 +191,7 @@ void parseArgument(int32_t argc, char *argv[]) { pPrint("%s totalRowsOfT2:%d %s", GREEN, g_stConfInfo.totalRowsOfT2, NC); pPrint("%s startTimestamp:%" PRId64" %s", GREEN, g_stConfInfo.startTimestamp, NC); pPrint("%s showMsgFlag:%d %s", GREEN, g_stConfInfo.showMsgFlag, NC); +#endif } static int running = 1; @@ -267,7 +275,7 @@ int32_t init_env() { taos_free_result(pRes); // create row value - g_pRowValue = (char*)calloc(1, g_stConfInfo.numOfColumn * 16 + 128); + g_pRowValue = (char*)taosMemoryCalloc(1, g_stConfInfo.numOfColumn * 16 + 128); if (NULL == g_pRowValue) { return -1; } @@ -429,15 +437,21 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics, int32_t totalMsgs, int64_t walLog double consumeTime = (double)(endTime - startTime) / 1000000; if (batchCnt != totalMsgs) { - pPrint("%s inserted msgs: %d and consume msgs: %d mismatch %s", GREEN, totalMsgs, batchCnt, NC); + printf("%s inserted msgs: %d and consume msgs: %d mismatch %s", GREEN, totalMsgs, batchCnt, NC); + exit(-1); } - pPrint("consume result: msgs: %d, skip log cnt: %d, time used:%.3f second\n", batchCnt, skipLogNum, consumeTime); + if (0 == g_stConfInfo.simCase) { + printf("consume result: msgs: %d, skip log cnt: %d, time used:%.3f second\n", batchCnt, skipLogNum, consumeTime); + } else { + printf("{consume success: %d}", totalMsgs); + } taosFprintfFile(g_fp, "|%10d | %10.3f | %8.2f | %10.2f| %10.2f |\n", batchCnt, consumeTime, (double)batchCnt / consumeTime, (double)walLogSize / (1024 * 1024.0) / consumeTime, (double)walLogSize / 1024.0 / batchCnt); err = tmq_consumer_close(tmq); if (err) { fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(err)); + exit(-1); } } @@ -458,7 +472,7 @@ int32_t syncWriteData() { taos_free_result(pRes); char* buffer = NULL; - buffer = (char*)malloc(MAX_SQL_STR_LEN); + buffer = (char*)taosMemoryMalloc(MAX_SQL_STR_LEN); if (NULL == buffer) { return -1; } @@ -491,7 +505,7 @@ int32_t syncWriteData() { int code = queryDB(pConn, buffer); if (0 != code){ fprintf(stderr, "insert data error!\n"); - tfree(buffer); + taosMemoryFreeClear(buffer); return -1; } @@ -503,7 +517,7 @@ int32_t syncWriteData() { } } } - tfree(buffer); + taosMemoryFreeClear(buffer); return totalMsgs; } @@ -525,7 +539,7 @@ int32_t syncWriteDataByRatio() { taos_free_result(pRes); char* buffer = NULL; - buffer = (char*)malloc(MAX_SQL_STR_LEN); + buffer = (char*)taosMemoryMalloc(MAX_SQL_STR_LEN); if (NULL == buffer) { return -1; } @@ -583,7 +597,7 @@ int32_t syncWriteDataByRatio() { int code = queryDB(pConn, buffer); if (0 != code){ fprintf(stderr, "insert data error!\n"); - tfree(buffer); + taosMemoryFreeClear(buffer); return -1; } @@ -597,7 +611,7 @@ int32_t syncWriteDataByRatio() { } } pPrint("expect insert rows: T1[%d] T2[%d], actual insert rows: T1[%d] T2[%d]\n", g_stConfInfo.totalRowsOfPerTbl, g_stConfInfo.totalRowsOfT2, insertedOfT1, insertedOfT2); - tfree(buffer); + taosMemoryFreeClear(buffer); return totalMsgs; } @@ -679,12 +693,17 @@ int main(int32_t argc, char *argv[]) { walLogSize = getDirectorySize(g_stConfInfo.vnodeWalPath); if (walLogSize <= 0) { - pError("vnode2/wal size incorrect!"); + printf("vnode2/wal size incorrect!"); + exit(-1); } else { - pPrint(".log file size in vnode2/wal: %.3f MBytes\n", (double)walLogSize/(1024 * 1024.0)); + if (0 == g_stConfInfo.simCase) { + pPrint(".log file size in vnode2/wal: %.3f MBytes\n", (double)walLogSize/(1024 * 1024.0)); + } } - - pPrint("insert result: %d rows, %d msgs, time:%.3f sec, speed:%.1f rows/second, %.1f msgs/second\n", totalRows, totalMsgs, seconds, rowsSpeed, msgsSpeed); + + if (0 == g_stConfInfo.simCase) { + pPrint("insert result: %d rows, %d msgs, time:%.3f sec, speed:%.1f rows/second, %.1f msgs/second\n", totalRows, totalMsgs, seconds, rowsSpeed, msgsSpeed); + } taosFprintfFile(g_fp, "|%10d | %10.3f | %8.2f | %10.3f ", totalMsgs, seconds, msgsSpeed, (double)walLogSize/(1024 * 1024.0)); } @@ -700,7 +719,7 @@ int main(int32_t argc, char *argv[]) { perf_loop(tmq, topic_list, totalMsgs, walLogSize); - tfree(g_pRowValue); + taosMemoryFreeClear(g_pRowValue); taosFprintfFile(g_fp, "\n"); taosCloseFile(&g_fp); return 0; diff --git a/tests/tsim/src/simParse.c b/tests/tsim/src/simParse.c index 7a1950611eafd3c84cc7c2a34ecc17a473ba37c3..a0721941e34afe398146165f71b0d9c6141792b6 100644 --- a/tests/tsim/src/simParse.c +++ b/tests/tsim/src/simParse.c @@ -156,17 +156,17 @@ SScript *simBuildScriptObj(char *fileName) { if (cmdLine[i].jump == 0) cmdLine[i].jump = numOfLines; } - SScript *script = malloc(sizeof(SScript)); + SScript *script = taosMemoryMalloc(sizeof(SScript)); memset(script, 0, sizeof(SScript)); script->type = SIM_SCRIPT_TYPE_MAIN; script->numOfLines = numOfLines; tstrncpy(script->fileName, fileName, sizeof(script->fileName)); - script->optionBuffer = malloc(optionOffset); + script->optionBuffer = taosMemoryMalloc(optionOffset); memcpy(script->optionBuffer, optionBuffer, optionOffset); - script->lines = malloc(sizeof(SCmdLine) * numOfLines); + script->lines = taosMemoryMalloc(sizeof(SCmdLine) * numOfLines); memcpy(script->lines, cmdLine, sizeof(SCmdLine) * numOfLines); return script; @@ -239,7 +239,7 @@ SScript *simParseScript(char *fileName) { return NULL; } } - if(buffer != NULL) free(buffer); + if(buffer != NULL) taosMemoryFree(buffer); taosCloseFile(&pFile); script = simBuildScriptObj(fileName); diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index eddcd3afc226c934c7a310ca5cd9b2bab3a3544f..eb5fb682642dce2601167f671370d9a2320c3b8b 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -60,16 +60,16 @@ void simFreeScript(SScript *script) { simDebug("script:%s, background thread joined", bgScript->fileName); taos_close(bgScript->taos); - tfree(bgScript->lines); - tfree(bgScript->optionBuffer); - tfree(bgScript); + taosMemoryFreeClear(bgScript->lines); + taosMemoryFreeClear(bgScript->optionBuffer); + taosMemoryFreeClear(bgScript); } simDebug("script:%s, is cleaned", script->fileName); taos_close(script->taos); - tfree(script->lines); - tfree(script->optionBuffer); - tfree(script); + taosMemoryFreeClear(script->lines); + taosMemoryFreeClear(script->optionBuffer); + taosMemoryFreeClear(script); } } diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index dae0a1c84023a649cfd14b9a188db46537e64882..f064833691a36e9ef3512d1648ea9a5934ab78e9 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -4,6 +4,7 @@ IF (TD_TAOS_TOOLS) INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include/common) INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include/util) INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include/os) + INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include/libs/transport) ADD_SUBDIRECTORY(taos-tools) ENDIF () diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt index b351675e4784a2ce31f321d8e224dc802fef1399..284693795ee471ad2d631758970c3033dc8e0c6c 100644 --- a/tools/shell/CMakeLists.txt +++ b/tools/shell/CMakeLists.txt @@ -4,9 +4,7 @@ add_executable(shell ${SHELL_SRC}) target_link_libraries( shell PUBLIC taos - PUBLIC util - PUBLIC common - PUBLIC os + PRIVATE os common transport util ) target_include_directories( shell diff --git a/tools/shell/inc/syncMsg.h b/tools/shell/inc/syncMsg.h new file mode 100644 index 0000000000000000000000000000000000000000..85ac9c78affa5282d5ca703caffc1bc5c24461bb --- /dev/null +++ b/tools/shell/inc/syncMsg.h @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_SYNC_MSG_H +#define TDENGINE_SYNC_MSG_H + +#ifdef __cplusplus +extern "C" { +#endif +#include "tsync.h" + +typedef enum { + TAOS_SMSG_START = 0, + TAOS_SMSG_SYNC_DATA = 1, + TAOS_SMSG_SYNC_DATA_RSP = 2, + TAOS_SMSG_SYNC_FWD = 3, + TAOS_SMSG_SYNC_FWD_RSP = 4, + TAOS_SMSG_SYNC_REQ = 5, + TAOS_SMSG_SYNC_REQ_RSP = 6, + TAOS_SMSG_SYNC_MUST = 7, + TAOS_SMSG_SYNC_MUST_RSP = 8, + TAOS_SMSG_STATUS = 9, + TAOS_SMSG_STATUS_RSP = 10, + TAOS_SMSG_SETUP = 11, + TAOS_SMSG_SETUP_RSP = 12, + TAOS_SMSG_SYNC_FILE = 13, + TAOS_SMSG_SYNC_FILE_RSP = 14, + TAOS_SMSG_TEST = 15, + TAOS_SMSG_END = 16 +} ESyncMsgType; + +typedef enum { + SYNC_STATUS_BROADCAST, + SYNC_STATUS_BROADCAST_RSP, + SYNC_STATUS_SETUP_CONN, + SYNC_STATUS_SETUP_CONN_RSP, + SYNC_STATUS_EXCHANGE_DATA, + SYNC_STATUS_EXCHANGE_DATA_RSP, + SYNC_STATUS_CHECK_ROLE, + SYNC_STATUS_CHECK_ROLE_RSP +} ESyncStatusType; + +#pragma pack(push, 1) + +typedef struct { + int8_t type; // msg type + int8_t protocol; // protocol version + uint16_t signature; // fixed value + int32_t code; // + int32_t cId; // cluster Id + int32_t vgId; // vg ID + int32_t len; // content length, does not include head + uint32_t cksum; +} SSyncHead; + +typedef struct { + SSyncHead head; + uint16_t port; + uint16_t tranId; + int32_t sourceId; // only for arbitrator + char fqdn[TSDB_FQDN_LEN]; +} SSyncMsg; + +typedef struct { + SSyncHead head; + int8_t sync; + int8_t reserved; + uint16_t tranId; + int8_t reserverd[4]; +} SSyncRsp; + +typedef struct { + int8_t role; + uint64_t version; +} SPeerStatus; + +typedef struct { + SSyncHead head; + int8_t role; + int8_t ack; + int8_t type; + int8_t reserved[3]; + uint16_t tranId; + uint64_t version; + SPeerStatus peersStatus[TAOS_SYNC_MAX_REPLICA]; +} SPeersStatus; + +typedef struct { + SSyncHead head; + uint64_t fversion; +} SFileVersion; + +typedef struct { + SSyncHead head; + int8_t ack; +} SFileAck; + +typedef struct { + SSyncHead head; + uint64_t version; + int32_t code; +} SFwdRsp; + +#pragma pack(pop) + +#define SYNC_PROTOCOL_VERSION 1 +#define SYNC_SIGNATURE ((uint16_t)(0xCDEF)) + +extern char *statusType[]; + +uint16_t syncGenTranId(); +int32_t syncCheckHead(SSyncHead *pHead); + +void syncBuildSyncFwdMsg(SSyncHead *pHead, int32_t vgId, int32_t len); +void syncBuildSyncFwdRsp(SFwdRsp *pMsg, int32_t vgId, uint64_t version, int32_t code); +void syncBuildSyncReqMsg(SSyncMsg *pMsg, int32_t vgId); +void syncBuildSyncDataMsg(SSyncMsg *pMsg, int32_t vgId); +void syncBuildSyncSetupMsg(SSyncMsg *pMsg, int32_t vgId); +void syncBuildPeersStatus(SPeersStatus *pMsg, int32_t vgId); +void syncBuildSyncTestMsg(SSyncMsg *pMsg, int32_t vgId); + +void syncBuildFileAck(SFileAck *pMsg, int32_t vgId); +void syncBuildFileVersion(SFileVersion *pMsg, int32_t vgId); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_VNODEPEER_H diff --git a/tools/shell/inc/tsync.h b/tools/shell/inc/tsync.h new file mode 100644 index 0000000000000000000000000000000000000000..d1b68e3f5a1e27b63dd08a4d1d4862c7b1e68179 --- /dev/null +++ b/tools/shell/inc/tsync.h @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_SYNC_H +#define TDENGINE_SYNC_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define TAOS_SYNC_MAX_REPLICA 5 +#define TAOS_SYNC_MAX_INDEX 0x7FFFFFFF + +typedef enum { + TAOS_SYNC_ROLE_OFFLINE = 0, + TAOS_SYNC_ROLE_UNSYNCED = 1, + TAOS_SYNC_ROLE_SYNCING = 2, + TAOS_SYNC_ROLE_SLAVE = 3, + TAOS_SYNC_ROLE_MASTER = 4 +} ESyncRole; + +typedef enum { + TAOS_SYNC_STATUS_INIT = 0, + TAOS_SYNC_STATUS_START = 1, + TAOS_SYNC_STATUS_FILE = 2, + TAOS_SYNC_STATUS_CACHE = 3 +} ESyncStatus; + +typedef struct { + uint32_t nodeId; // node ID assigned by TDengine + uint16_t nodePort; // node sync Port + char nodeFqdn[TSDB_FQDN_LEN]; // node FQDN +} SNodeInfo; + +typedef struct { + int8_t quorum; // number of confirms required, >=1 + int8_t replica; // number of replications, >=1 + SNodeInfo nodeInfo[TAOS_SYNC_MAX_REPLICA]; +} SSyncCfg; + +typedef struct { + int32_t selfIndex; + uint32_t nodeId[TAOS_SYNC_MAX_REPLICA]; + int32_t role[TAOS_SYNC_MAX_REPLICA]; +} SNodesRole; + +// get the wal file from index or after +// return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file +typedef int32_t (*FGetWalInfo)(int32_t vgId, char *fileName, int64_t *fileId); + +// when a forward pkt is received, call this to handle data +typedef int32_t (*FWriteToCache)(int32_t vgId, void *pHead, int32_t qtype, void *pMsg); + +// when forward is confirmed by peer, master call this API to notify app +typedef void (*FConfirmForward)(int32_t vgId, void *mhandle, int32_t code); + +// when role is changed, call this to notify app +typedef void (*FNotifyRole)(int32_t vgId, int8_t role); + +// if a number of retrieving data failed, call this to start flow control +typedef void (*FNotifyFlowCtrl)(int32_t vgId, int32_t level); + +// when data file is synced successfully, notity app +typedef void (*FStartSyncFile)(int32_t vgId); +typedef void (*FStopSyncFile)(int32_t vgId, uint64_t fversion); + +// get file version +typedef int32_t (*FGetVersion)(int32_t vgId, uint64_t *fver, uint64_t *vver); + +typedef int32_t (*FSendFile)(void *tsdb, SOCKET socketFd); +typedef int32_t (*FRecvFile)(void *tsdb, SOCKET socketFd); + +typedef struct { + int32_t vgId; // vgroup ID + uint64_t version; // initial version + SSyncCfg syncCfg; // configuration from mgmt + char path[TSDB_FILENAME_LEN]; // path to the file + void * pTsdb; + FGetWalInfo getWalInfoFp; + FWriteToCache writeToCacheFp; + FConfirmForward confirmForward; + FNotifyRole notifyRoleFp; + FNotifyFlowCtrl notifyFlowCtrlFp; + FStartSyncFile startSyncFileFp; + FStopSyncFile stopSyncFileFp; + FGetVersion getVersionFp; + FSendFile sendFileFp; + FRecvFile recvFileFp; +} SSyncInfo; + +typedef void *tsync_h; + +int32_t syncInit(); +void syncCleanUp(); + +int64_t syncStart(const SSyncInfo *); +void syncStop(int64_t rid); +int32_t syncReconfig(int64_t rid, const SSyncCfg *); +int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int32_t qtype, bool force); +void syncConfirmForward(int64_t rid, uint64_t version, int32_t code, bool force); +void syncRecover(int64_t rid); // recover from other nodes: +int32_t syncGetNodesRole(int64_t rid, SNodesRole *); + +extern char *syncRole[]; + +//global configurable parameters +extern int32_t sDebugFlag; +extern char tsArbitrator[]; +extern uint16_t tsSyncPort; + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_SYNC_H diff --git a/tools/shell/src/backup/shellCheck.c b/tools/shell/src/backup/shellCheck.c index 919cb0d84636dec2579b787288820fe2e803c319..dc18ecd3f8d877a4bb158cffee9a6cda02edd221 100644 --- a/tools/shell/src/backup/shellCheck.c +++ b/tools/shell/src/backup/shellCheck.c @@ -72,7 +72,7 @@ static int32_t shellShowTables(TAOS *con, char *db) { int32_t tbIndex = tbNum++; if (tbMallocNum < tbNum) { tbMallocNum = (tbMallocNum * 2 + 1); - char** tbNames1 = realloc(tbNames, tbMallocNum * sizeof(char *)); + char** tbNames1 = taosMemoryRealloc(tbNames, tbMallocNum * sizeof(char *)); if (tbNames1 == NULL) { fprintf(stdout, "failed to malloc tablenames, num:%d\n", tbMallocNum); code = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -81,7 +81,7 @@ static int32_t shellShowTables(TAOS *con, char *db) { tbNames = tbNames1; } - tbNames[tbIndex] = malloc(TSDB_TABLE_NAME_LEN); + tbNames[tbIndex] = taosMemoryMalloc(TSDB_TABLE_NAME_LEN); strncpy(tbNames[tbIndex], (const char *)row[0], TSDB_TABLE_NAME_LEN); if (tbIndex % 100000 == 0 && tbIndex != 0) { fprintf(stdout, "%d tablenames fetched\n", tbIndex); @@ -97,9 +97,9 @@ static int32_t shellShowTables(TAOS *con, char *db) { static void shellFreeTbnames() { for (int32_t i = 0; i < tbNum; ++i) { - free(tbNames[i]); + taosMemoryFree(tbNames[i]); } - free(tbNames); + taosMemoryFree(tbNames); } static void *shellCheckThreadFp(void *arg) { @@ -153,7 +153,7 @@ static void *shellCheckThreadFp(void *arg) { static void shellRunCheckThreads(TAOS *con, SShellArguments *_args) { TdThreadAttr thattr; - ShellThreadObj *threadObj = (ShellThreadObj *)calloc(_args->threadNum, sizeof(ShellThreadObj)); + ShellThreadObj *threadObj = (ShellThreadObj *)taosMemoryCalloc(_args->threadNum, sizeof(ShellThreadObj)); for (int t = 0; t < _args->threadNum; ++t) { ShellThreadObj *pThread = threadObj + t; pThread->threadIndex = t; @@ -177,7 +177,7 @@ static void shellRunCheckThreads(TAOS *con, SShellArguments *_args) { for (int t = 0; t < _args->threadNum; ++t) { taos_close(threadObj[t].taos); } - free(threadObj); + taosMemoryFree(threadObj); } void shellCheck(TAOS *con, SShellArguments *_args) { diff --git a/tools/shell/src/backup/shellDarwin.c b/tools/shell/src/backup/shellDarwin.c index 69c7a7bc4e568f9f63a0b63be912f41be7adb8bb..93335776bad446d4ad9575efa838ed1723421901 100644 --- a/tools/shell/src/backup/shellDarwin.c +++ b/tools/shell/src/backup/shellDarwin.c @@ -206,8 +206,8 @@ int32_t shellReadCommand(TAOS *con, char *command) { char utf8_array[10] = "\0"; Command cmd; memset(&cmd, 0, sizeof(cmd)); - cmd.buffer = (char *)calloc(1, MAX_COMMAND_SIZE); - cmd.command = (char *)calloc(1, MAX_COMMAND_SIZE); + cmd.buffer = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); + cmd.command = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); showOnScreen(&cmd); // Read input. @@ -252,8 +252,8 @@ int32_t shellReadCommand(TAOS *con, char *command) { printf("\n"); if (isReadyGo(&cmd)) { sprintf(command, "%s%s", cmd.buffer, cmd.command); - tfree(cmd.buffer); - tfree(cmd.command); + taosMemoryFreeClear(cmd.buffer); + taosMemoryFreeClear(cmd.command); return 0; } else { updateBuffer(&cmd); @@ -367,7 +367,7 @@ void *shellLoopQuery(void *arg) { taosThreadCleanupPush(cleanup_handler, NULL); - char *command = malloc(MAX_COMMAND_SIZE); + char *command = taosMemoryMalloc(MAX_COMMAND_SIZE); if (command == NULL){ tscError("failed to malloc command"); return NULL; @@ -386,7 +386,7 @@ void *shellLoopQuery(void *arg) { resetTerminalMode(); } while (shellRunCommand(con, command) == 0); - tfree(command); + taosMemoryFreeClear(command); exitShell(); taosThreadCleanupPop(1); @@ -429,7 +429,7 @@ void showOnScreen(Command *cmd) { int size = 0; // Print out the command. - char *total_string = malloc(MAX_COMMAND_SIZE); + char *total_string = taosMemoryMalloc(MAX_COMMAND_SIZE); memset(total_string, '\0', MAX_COMMAND_SIZE); if (strcmp(cmd->buffer, "") == 0) { sprintf(total_string, "%s%s", PROMPT_HEADER, cmd->command); @@ -461,7 +461,7 @@ void showOnScreen(Command *cmd) { str = total_string + size; } - free(total_string); + taosMemoryFree(total_string); /* for (int i = 0; i < size; i++){ */ /* char c = total_string[i]; */ /* if (k % w.ws_col == 0) { */ diff --git a/tools/shell/src/backup/shellImport.c b/tools/shell/src/backup/shellImport.c index 398bcf2280f5946bf1c2d5424813c8a507ed9a5b..130c72a20bbdb831fb1fe3fa8d62ca93f5aa45dd 100644 --- a/tools/shell/src/backup/shellImport.c +++ b/tools/shell/src/backup/shellImport.c @@ -97,9 +97,9 @@ static void shellCheckTablesSQLFile(const char *directoryName) static void shellMallocSQLFiles() { - shellSQLFiles = (char**)calloc(shellSQLFileNum, sizeof(char*)); + shellSQLFiles = (char**)taosMemoryCalloc(shellSQLFileNum, sizeof(char*)); for (int i = 0; i < shellSQLFileNum; i++) { - shellSQLFiles[i] = calloc(1, TSDB_FILENAME_LEN); + shellSQLFiles[i] = taosMemoryCalloc(1, TSDB_FILENAME_LEN); } } @@ -130,20 +130,20 @@ static void shellGetDirectoryFileList(char *inputDir) static void shellSourceFile(TAOS *con, char *fptr) { wordexp_t full_path; int read_len = 0; - char * cmd = malloc(tsMaxSQLStringLen); + char * cmd = taosMemoryMalloc(tsMaxSQLStringLen); size_t cmd_len = 0; char * line = NULL; if (wordexp(fptr, &full_path, 0) != 0) { fprintf(stderr, "ERROR: illegal file name\n"); - free(cmd); + taosMemoryFree(cmd); return; } char *fname = full_path.we_wordv[0]; if (fname == NULL) { fprintf(stderr, "ERROR: invalid filename\n"); - free(cmd); + taosMemoryFree(cmd); return; } @@ -152,7 +152,7 @@ static void shellSourceFile(TAOS *con, char *fptr) { fprintf(stderr, "ERROR: file %s is not exist\n", fptr); wordfree(&full_path); - free(cmd); + taosMemoryFree(cmd); return; } @@ -160,7 +160,7 @@ static void shellSourceFile(TAOS *con, char *fptr) { fprintf(stderr, "ERROR: file %s is not readable\n", fptr); wordfree(&full_path); - free(cmd); + taosMemoryFree(cmd); return; } */ @@ -170,7 +170,7 @@ static void shellSourceFile(TAOS *con, char *fptr) { if (pFile == NULL) { fprintf(stderr, "ERROR: failed to open file %s\n", fname); wordfree(&full_path); - free(cmd); + taosMemoryFree(cmd); return; } @@ -209,8 +209,8 @@ static void shellSourceFile(TAOS *con, char *fptr) { cmd_len = 0; } - free(cmd); - if(line != NULL) free(line); + taosMemoryFree(cmd); + if(line != NULL) taosMemoryFree(line); wordfree(&full_path); taosCloseFile(&pFile); } @@ -233,7 +233,7 @@ void* shellImportThreadFp(void *arg) static void shellRunImportThreads(SShellArguments* _args) { TdThreadAttr thattr; - ShellThreadObj *threadObj = (ShellThreadObj *)calloc(_args->threadNum, sizeof(ShellThreadObj)); + ShellThreadObj *threadObj = (ShellThreadObj *)taosMemoryCalloc(_args->threadNum, sizeof(ShellThreadObj)); for (int t = 0; t < _args->threadNum; ++t) { ShellThreadObj *pThread = threadObj + t; pThread->threadIndex = t; @@ -260,7 +260,7 @@ static void shellRunImportThreads(SShellArguments* _args) for (int t = 0; t < _args->threadNum; ++t) { taos_close(threadObj[t].taos); } - free(threadObj); + taosMemoryFree(threadObj); } void source_dir(TAOS* con, SShellArguments* _args) { diff --git a/tools/shell/src/backup/shellWindows.c b/tools/shell/src/backup/shellWindows.c index 1244c7b06022a53380c1a84304a4a2cdb76a17d2..92ac7fd7213b13b34d94714b32216b1fb0435477 100644 --- a/tools/shell/src/backup/shellWindows.c +++ b/tools/shell/src/backup/shellWindows.c @@ -239,7 +239,7 @@ void updateBuffer(Command *cmd) { } int isReadyGo(Command *cmd) { - char *total = malloc(MAX_COMMAND_SIZE); + char *total = taosMemoryMalloc(MAX_COMMAND_SIZE); memset(total, 0, MAX_COMMAND_SIZE); sprintf(total, "%s%s", cmd->buffer, cmd->command); @@ -247,11 +247,11 @@ int isReadyGo(Command *cmd) { "(^.*;\\s*$)|(^\\s*$)|(^\\s*exit\\s*$)|(^\\s*q\\s*$)|(^\\s*quit\\s*$)|(^" "\\s*clear\\s*$)"; if (regex_match(total, reg_str, REG_EXTENDED | REG_ICASE)) { - free(total); + taosMemoryFree(total); return 1; } - free(total); + taosMemoryFree(total); return 0; } @@ -268,8 +268,8 @@ void insertChar(Command *cmd, char c) { int32_t shellReadCommand(TAOS *con, char command[]) { Command cmd; memset(&cmd, 0, sizeof(cmd)); - cmd.buffer = (char *)calloc(1, MAX_COMMAND_SIZE); - cmd.command = (char *)calloc(1, MAX_COMMAND_SIZE); + cmd.buffer = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); + cmd.command = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); // Read input. char c; @@ -281,9 +281,9 @@ int32_t shellReadCommand(TAOS *con, char command[]) { case '\r': if (isReadyGo(&cmd)) { sprintf(command, "%s%s", cmd.buffer, cmd.command); - free(cmd.buffer); + taosMemoryFree(cmd.buffer); cmd.buffer = NULL; - free(cmd.command); + taosMemoryFree(cmd.command); cmd.command = NULL; return 0; } else { @@ -301,7 +301,7 @@ int32_t shellReadCommand(TAOS *con, char command[]) { void *shellLoopQuery(void *arg) { TAOS *con = (TAOS *)arg; - char *command = malloc(MAX_COMMAND_SIZE); + char *command = taosMemoryMalloc(MAX_COMMAND_SIZE); if (command == NULL) return NULL; int32_t err = 0; diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index fd993998b823411c666194d9f2bb9c6390baa2c4..7cbadfaf5b015a3f6ef039df1370c9381238ef99 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -230,7 +230,7 @@ void updateBuffer(Command *cmd) { int isReadyGo(Command *cmd) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); - char *total = (char *)calloc(1, MAX_COMMAND_SIZE); + char *total = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); memset(cmd->command + cmd->commandSize, 0, MAX_COMMAND_SIZE - cmd->commandSize); sprintf(total, "%s%s", cmd->buffer, cmd->command); @@ -238,20 +238,20 @@ int isReadyGo(Command *cmd) { "(^.*;\\s*$)|(^\\s*$)|(^\\s*exit\\s*$)|(^\\s*q\\s*$)|(^\\s*quit\\s*$)|(^" "\\s*clear\\s*$)"; if (regex_match(total, reg_str, REG_EXTENDED | REG_ICASE)) { - free(total); + taosMemoryFree(total); return 1; } - free(total); + taosMemoryFree(total); return 0; } void getMbSizeInfo(const char *str, int *size, int *width) { - TdWchar *wc = (TdWchar *)calloc(sizeof(TdWchar), MAX_COMMAND_SIZE); + TdWchar *wc = (TdWchar *)taosMemoryCalloc(sizeof(TdWchar), MAX_COMMAND_SIZE); *size = strlen(str); taosMbsToWchars(wc, str, MAX_COMMAND_SIZE); *width = taosWcharsWidth(wc, MAX_COMMAND_SIZE); - free(wc); + taosMemoryFree(wc); } void resetCommand(Command *cmd, const char s[]) { diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 1b35afb57d231c07181499fa43c66cbad137e72c..0de663f2790ecde7e9c239df7cb4d77b614c944c 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -210,7 +210,7 @@ int32_t shellRunCommand(TAOS *con, char *command) { history.hist[(history.hend + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE] == NULL || strcmp(command, history.hist[(history.hend + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE]) != 0) { if (history.hist[history.hend] != NULL) { - tfree(history.hist[history.hend]); + taosMemoryFreeClear(history.hist[history.hend]); } history.hist[history.hend] = strdup(command); @@ -925,7 +925,7 @@ void read_history() { } } - if(line != NULL) free(line); + if(line != NULL) taosMemoryFree(line); taosCloseFile(&pFile); } @@ -945,7 +945,7 @@ void write_history() { for (int i = history.hstart; i != history.hend;) { if (history.hist[i] != NULL) { taosFprintfFile(pFile, "%s\n", history.hist[i]); - tfree(history.hist[i]); + taosMemoryFreeClear(history.hist[i]); } i = (i + 1) % MAX_HISTORY_SIZE; } @@ -968,13 +968,13 @@ int isCommentLine(char *line) { void source_file(TAOS *con, char *fptr) { wordexp_t full_path; int read_len = 0; - char *cmd = calloc(1, TSDB_MAX_ALLOWED_SQL_LEN + 1); + char *cmd = taosMemoryCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN + 1); size_t cmd_len = 0; char *line = NULL; if (wordexp(fptr, &full_path, 0) != 0) { fprintf(stderr, "ERROR: illegal file name\n"); - free(cmd); + taosMemoryFree(cmd); return; } @@ -985,7 +985,7 @@ void source_file(TAOS *con, char *fptr) { fprintf(stderr, "ERROR: file %s is not exist\n", fptr); wordfree(&full_path); - free(cmd); + taosMemoryFree(cmd); return; } */ @@ -995,7 +995,7 @@ void source_file(TAOS *con, char *fptr) { if (pFile == NULL) { fprintf(stderr, "ERROR: failed to open file %s\n", fname); wordfree(&full_path); - free(cmd); + taosMemoryFree(cmd); return; } @@ -1021,8 +1021,8 @@ void source_file(TAOS *con, char *fptr) { cmd_len = 0; } - free(cmd); - if(line != NULL) free(line); + taosMemoryFree(cmd); + if(line != NULL) taosMemoryFree(line); wordfree(&full_path); taosCloseFile(&pFile); } diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index 78d6f74df1666fac42db073a82f52e912145cbd4..70563c79e697eee201798926f69f1af68244af20 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -17,6 +17,7 @@ #include "os.h" #include "shell.h" #include "tglobal.h" +#include "tconfig.h" #include "shellCommand.h" #include "tbase64.h" #include "tlog.h" @@ -32,6 +33,8 @@ int indicator = 1; void insertChar(Command *cmd, char *c, int size); +void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, + int32_t pkgNum, char *pkgType); const char *argp_program_version = version; const char *argp_program_bug_address = ""; static char doc[] = ""; @@ -56,10 +59,11 @@ static struct argp_option options[] = { {"check", 'k', "CHECK", 0, "Check tables."}, {"database", 'd', "DATABASE", 0, "Database to use when connecting to the server."}, {"timezone", 'z', "TIMEZONE", 0, "Time zone of the shell, default is local."}, - {"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync|speen|fqdn."}, + {"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync|speed|fqdn."}, {"pktlen", 'l', "PKTLEN", 0, "Packet length used for net test, default is 1000 bytes."}, {"pktnum", 'N', "PKTNUM", 0, "Packet numbers used for net test, default is 100."}, - {"pkttype", 'S', "PKTTYPE", 0, "Packet type used for net test, default is TCP."}, +// Shuduo: 3.0 does not support UDP any more +// {"pkttype", 'S', "PKTTYPE", 0, "Packet type used for net test, default is TCP."}, {0}}; static error_t parse_opt(int key, char *arg, struct argp_state *state) { @@ -236,8 +240,8 @@ int32_t shellReadCommand(TAOS *con, char *command) { char utf8_array[10] = "\0"; Command cmd; memset(&cmd, 0, sizeof(cmd)); - cmd.buffer = (char *)calloc(1, MAX_COMMAND_SIZE); - cmd.command = (char *)calloc(1, MAX_COMMAND_SIZE); + cmd.buffer = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); + cmd.command = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); showOnScreen(&cmd); // Read input. @@ -286,8 +290,8 @@ int32_t shellReadCommand(TAOS *con, char *command) { printf("\n"); if (isReadyGo(&cmd)) { sprintf(command, "%s%s", cmd.buffer, cmd.command); - tfree(cmd.buffer); - tfree(cmd.command); + taosMemoryFreeClear(cmd.buffer); + taosMemoryFreeClear(cmd.command); return 0; } else { updateBuffer(&cmd); @@ -401,7 +405,7 @@ void *shellLoopQuery(void *arg) { taosThreadCleanupPush(cleanup_handler, NULL); - char *command = malloc(MAX_COMMAND_SIZE); + char *command = taosMemoryMalloc(MAX_COMMAND_SIZE); if (command == NULL){ uError("failed to malloc command"); return NULL; @@ -420,7 +424,7 @@ void *shellLoopQuery(void *arg) { resetTerminalMode(); } while (shellRunCommand(con, command) == 0); - tfree(command); + taosMemoryFreeClear(command); exitShell(); taosThreadCleanupPop(1); @@ -463,7 +467,7 @@ void showOnScreen(Command *cmd) { int size = 0; // Print out the command. - char *total_string = malloc(MAX_COMMAND_SIZE); + char *total_string = taosMemoryMalloc(MAX_COMMAND_SIZE); memset(total_string, '\0', MAX_COMMAND_SIZE); if (strcmp(cmd->buffer, "") == 0) { sprintf(total_string, "%s%s", PROMPT_HEADER, cmd->command); @@ -495,7 +499,7 @@ void showOnScreen(Command *cmd) { str = total_string + size; } - free(total_string); + taosMemoryFree(total_string); /* for (int i = 0; i < size; i++){ */ /* char c = total_string[i]; */ /* if (k % w.ws_col == 0) { */ @@ -616,29 +620,35 @@ int main(int argc, char *argv[]) { shellParseArgument(argc, argv, &args); -#if 0 if (args.dump_config) { - taosInitGlobalCfg(); - taosReadGlobalLogCfg(); + taosInitCfg(configDir, NULL, NULL, NULL, 1); - if (taosReadGlobalCfg() ! =0) { - printf("TDengine read global config failed"); + SConfig *pCfg = taosGetCfg(); + if (NULL == pCfg) { + printf("TDengine read global config failed!\n"); exit(EXIT_FAILURE); } - - taosDumpGlobalCfg(); + cfgDumpCfg(pCfg, 0, 1); exit(0); } if (args.netTestRole && args.netTestRole[0] != 0) { - if (taos_init()) { + TAOS *con = NULL; + if (args.auth == NULL) { + con = taos_connect(args.host, args.user, args.password, args.database, args.port); + } else { + con = taos_connect_auth(args.host, args.user, args.auth, args.database, args.port); + } + +/* if (taos_init()) { printf("Failed to init taos"); exit(EXIT_FAILURE); } + */ taosNetTest(args.netTestRole, args.host, args.port, args.pktLen, args.pktNum, args.pktType); + taos_close(con); exit(0); } -#endif /* Initialize the shell */ TAOS *con = shellInit(&args); diff --git a/tools/shell/src/backup/tnettest.c b/tools/shell/src/tnettest.c similarity index 80% rename from tools/shell/src/backup/tnettest.c rename to tools/shell/src/tnettest.c index ee32bfb6be47ab6c6a841c081b3b2eb8d3dd1547..9be341225664694d0eb05bb7d76fcfcd1cbe0c30 100644 --- a/tools/shell/src/backup/tnettest.c +++ b/tools/shell/src/tnettest.c @@ -14,18 +14,20 @@ */ #define _DEFAULT_SOURCE +#define ALLOW_FORBID_FUNC #include "os.h" #include "taosdef.h" #include "tmsg.h" #include "taoserror.h" #include "tlog.h" #include "tglobal.h" -#include "tsocket.h" #include "trpc.h" #include "rpcHead.h" #include "tchecksum.h" #include "syncMsg.h" +#include "osSocket.h" + #define MAX_PKG_LEN (64 * 1000) #define MAX_SPEED_PKG_LEN (1024 * 1024 * 1024) #define MIN_SPEED_PKG_LEN 1024 @@ -33,7 +35,7 @@ #define MIN_SPEED_PKG_NUM 1 #define BUFFER_SIZE (MAX_PKG_LEN + 1024) -extern int32_t tsRpcMaxUdpSize; +extern int tsRpcMaxUdpSize; typedef struct { char * hostFqdn; @@ -71,15 +73,23 @@ static void *taosNetBindUdpPort(void *sarg) { return NULL; } - if (taosSetSockOpt(serverSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); + if (pSocket == NULL) { + taosCloseSocketNoCheck1(serverSocket); + return NULL; + } + pSocket->fd = serverSocket; + pSocket->refId = 0; + + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { uError("failed to set the send buffer size for UDP socket\n"); - taosCloseSocket(serverSocket); + taosCloseSocket(&pSocket); return NULL; } - if (taosSetSockOpt(serverSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { uError("failed to set the receive buffer size for UDP socket\n"); - taosCloseSocket(serverSocket); + taosCloseSocket(&pSocket); return NULL; } @@ -98,13 +108,13 @@ static void *taosNetBindUdpPort(void *sarg) { uInfo("UDP: recv:%d bytes from %s at %d", iDataNum, taosInetNtoa(clientAddr.sin_addr), port); if (iDataNum > 0) { - iDataNum = taosSendto(serverSocket, buffer, iDataNum, 0, (struct sockaddr *)&clientAddr, (int32_t)sin_size); + iDataNum = taosSendto(pSocket, buffer, iDataNum, 0, (struct sockaddr *)&clientAddr, (int32_t)sin_size); } uInfo("UDP: send:%d bytes to %s at %d", iDataNum, taosInetNtoa(clientAddr.sin_addr), port); } - taosCloseSocket(serverSocket); + taosCloseSocket(&pSocket); return NULL; } @@ -132,25 +142,35 @@ static void *taosNetBindTcpPort(void *sarg) { server_addr.sin_addr.s_addr = htonl(INADDR_ANY); int32_t reuse = 1; - if (taosSetSockOpt(serverSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); + if (pSocket == NULL) { + taosCloseSocketNoCheck1(serverSocket); + return NULL; + } + pSocket->fd = serverSocket; + pSocket->refId = 0; + + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { uError("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); - taosCloseSocket(serverSocket); + taosCloseSocket(&pSocket); return NULL; } if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { uError("failed to bind TCP port:%d since %s", port, strerror(errno)); + taosCloseSocket(&pSocket); return NULL; } - if (taosKeepTcpAlive(serverSocket) < 0) { + if (taosKeepTcpAlive(pSocket) < 0) { uError("failed to set tcp server keep-alive option since %s", strerror(errno)); - taosCloseSocket(serverSocket); + taosCloseSocket(&pSocket); return NULL; } if (listen(serverSocket, 10) < 0) { uError("failed to listen TCP port:%d since %s", port, strerror(errno)); + taosCloseSocket(&pSocket); return NULL; } @@ -163,26 +183,26 @@ static void *taosNetBindTcpPort(void *sarg) { continue; } - int32_t ret = taosReadMsg(client, buffer, pinfo->pktLen); + int32_t ret = taosReadMsg(pSocket, buffer, pinfo->pktLen); if (ret < 0 || ret != pinfo->pktLen) { uError("TCP: failed to read %d bytes at port:%d since %s", pinfo->pktLen, port, strerror(errno)); - taosCloseSocket(serverSocket); + taosCloseSocket(&pSocket); return NULL; } uInfo("TCP: read:%d bytes from %s at %d", pinfo->pktLen, taosInetNtoa(clientAddr.sin_addr), port); - ret = taosWriteMsg(client, buffer, pinfo->pktLen); + ret = taosWriteMsg(pSocket, buffer, pinfo->pktLen); if (ret < 0) { uError("TCP: failed to write %d bytes at %d since %s", pinfo->pktLen, port, strerror(errno)); - taosCloseSocket(serverSocket); + taosCloseSocket(&pSocket); return NULL; } uInfo("TCP: write:%d bytes to %s at %d", pinfo->pktLen, taosInetNtoa(clientAddr.sin_addr), port); } - taosCloseSocket(serverSocket); + taosCloseSocket(&pSocket); return NULL; } @@ -196,9 +216,17 @@ static int32_t taosNetCheckTcpPort(STestInfo *info) { } int32_t reuse = 1; - if (taosSetSockOpt(clientSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); + if (pSocket == NULL) { + taosCloseSocketNoCheck1(clientSocket); + return -1; + } + pSocket->fd = clientSocket; + pSocket->refId = 0; + + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { uError("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); - taosCloseSocket(clientSocket); + taosCloseSocket(&pSocket); return -1; } @@ -210,27 +238,30 @@ static int32_t taosNetCheckTcpPort(STestInfo *info) { if (connect(clientSocket, (struct sockaddr *)&serverAddr, sizeof(serverAddr)) < 0) { uError("TCP: failed to connect port %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno)); + taosCloseSocket(&pSocket); return -1; } - taosKeepTcpAlive(clientSocket); + taosKeepTcpAlive(pSocket); sprintf(buffer, "client send TCP pkg to %s:%d, content: 1122334455", taosIpStr(info->hostIp), info->port); sprintf(buffer + info->pktLen - 16, "1122334455667788"); - int32_t ret = taosWriteMsg(clientSocket, buffer, info->pktLen); + int32_t ret = taosWriteMsg(pSocket, buffer, info->pktLen); if (ret < 0) { uError("TCP: failed to write msg to %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno)); + taosCloseSocket(&pSocket); return -1; } - ret = taosReadMsg(clientSocket, buffer, info->pktLen); + ret = taosReadMsg(pSocket, buffer, info->pktLen); if (ret < 0) { uError("TCP: failed to read msg from %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno)); + taosCloseSocket(&pSocket); return -1; } - taosCloseSocket(clientSocket); + taosCloseSocket(&pSocket); return 0; } @@ -247,13 +278,23 @@ static int32_t taosNetCheckUdpPort(STestInfo *info) { return -1; } - if (taosSetSockOpt(clientSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); + if (pSocket == NULL) { + taosCloseSocketNoCheck1(clientSocket); + return -1; + } + pSocket->fd = clientSocket; + pSocket->refId = 0; + + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { uError("failed to set the send buffer size for UDP socket\n"); + taosCloseSocket(&pSocket); return -1; } - if (taosSetSockOpt(clientSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { uError("failed to set the receive buffer size for UDP socket\n"); + taosCloseSocket(&pSocket); return -1; } @@ -268,9 +309,10 @@ static int32_t taosNetCheckUdpPort(STestInfo *info) { socklen_t sin_size = sizeof(*(struct sockaddr *)&serverAddr); - iDataNum = taosSendto(clientSocket, buffer, info->pktLen, 0, (struct sockaddr *)&serverAddr, (int32_t)sin_size); + iDataNum = taosSendto(pSocket, buffer, info->pktLen, 0, (struct sockaddr *)&serverAddr, (int32_t)sin_size); if (iDataNum < 0 || iDataNum != info->pktLen) { uError("UDP: failed to perform sendto func since %s", strerror(errno)); + taosCloseSocket(&pSocket); return -1; } @@ -280,10 +322,11 @@ static int32_t taosNetCheckUdpPort(STestInfo *info) { if (iDataNum < 0 || iDataNum != info->pktLen) { uError("UDP: received ack:%d bytes(expect:%d) from port:%d since %s", iDataNum, info->pktLen, info->port, strerror(errno)); + taosCloseSocket(&pSocket); return -1; } - taosCloseSocket(clientSocket); + taosCloseSocket(&pSocket); return 0; } @@ -339,7 +382,7 @@ void *taosNetInitRpc(char *secretEncrypt, char spi) { } static int32_t taosNetCheckRpc(const char* serverFqdn, uint16_t port, uint16_t pktLen, char spi, SStartupReq *pStep) { - SRpcEpSet epSet; + SEpSet epSet; SRpcMsg reqMsg; SRpcMsg rspMsg; void * pRpcConn; @@ -352,11 +395,10 @@ static int32_t taosNetCheckRpc(const char* serverFqdn, uint16_t port, uint16_t p return TSDB_CODE_RPC_NETWORK_UNAVAIL; } - memset(&epSet, 0, sizeof(SRpcEpSet)); - epSet.inUse = 0; + memset(&epSet, 0, sizeof(SEpSet)); + strcpy(epSet.eps[0].fqdn, serverFqdn); + epSet.eps[0].port = port; epSet.numOfEps = 1; - epSet.port[0] = port; - strcpy(epSet.fqdn[0], serverFqdn); reqMsg.msgType = TDMT_DND_NETWORK_TEST; reqMsg.pCont = rpcMallocCont(pktLen); @@ -364,7 +406,7 @@ static int32_t taosNetCheckRpc(const char* serverFqdn, uint16_t port, uint16_t p reqMsg.code = 0; reqMsg.handle = NULL; // rpc handle returned to app reqMsg.ahandle = NULL; // app handle set by client - strcpy(reqMsg.pCont, "nettest"); + strcpy(reqMsg.pCont, "dnode-nettest"); rpcSendRecv(pRpcConn, &epSet, &reqMsg, &rspMsg); @@ -398,9 +440,9 @@ static int32_t taosNetParseStartup(SStartupReq *pCont) { static void taosNetTestStartup(char *host, int32_t port) { uInfo("check startup, host:%s port:%d\n", host, port); - SStartupReq *pStep = malloc(sizeof(SStartupReq)); + SStartupReq *pStep = taosMemoryMalloc(sizeof(SStartupReq)); while (1) { - int32_t code = taosNetCheckRpc(host, port + TSDB_PORT_DNODEDNODE, 20, 0, pStep); + int32_t code = taosNetCheckRpc(host, port, 20, 0, pStep); if (code > 0) { code = taosNetParseStartup(pStep); } @@ -415,7 +457,7 @@ static void taosNetTestStartup(char *host, int32_t port) { } } - free(pStep); + taosMemoryFree(pStep); } static void taosNetCheckSync(char *host, int32_t port) { @@ -425,8 +467,8 @@ static void taosNetCheckSync(char *host, int32_t port) { return; } - SOCKET connFd = taosOpenTcpClientSocket(ip, (uint16_t)port, 0); - if (connFd < 0) { + TdSocketPtr pSocket = taosOpenTcpClientSocket(ip, (uint16_t)port, 0); + if (pSocket == NULL) { uError("failed to create socket while test port:%d since %s", port, strerror(errno)); return; } @@ -443,63 +485,60 @@ static void taosNetCheckSync(char *host, int32_t port) { pHead->len = sizeof(SSyncMsg) - sizeof(SSyncHead); taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SSyncHead)); - if (taosWriteMsg(connFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { + if (taosWriteMsg(pSocket, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { uError("failed to test port:%d while send msg since %s", port, strerror(errno)); return; } - if (taosReadMsg(connFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { + if (taosReadMsg(pSocket, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { uError("failed to test port:%d while recv msg since %s", port, strerror(errno)); } uInfo("successed to test TCP port:%d", port); - taosCloseSocket(connFd); + taosCloseSocket(&pSocket); } static void taosNetTestRpc(char *host, int32_t startPort, int32_t pkgLen) { - int32_t endPort = startPort + TSDB_PORT_SYNC; char spi = 0; - uInfo("check rpc, host:%s startPort:%d endPort:%d pkgLen:%d\n", host, startPort, endPort, pkgLen); + uInfo("check rpc, host:%s Port:%d pkgLen:%d\n", host, startPort, pkgLen); - for (uint16_t port = startPort; port < endPort; port++) { - int32_t sendpkgLen; - if (pkgLen <= tsRpcMaxUdpSize) { + uint16_t port = startPort; + int32_t sendpkgLen; + if (pkgLen <= tsRpcMaxUdpSize) { sendpkgLen = tsRpcMaxUdpSize + 1000; - } else { + } else { sendpkgLen = pkgLen; - } + } - tsRpcForceTcp = 1; - int32_t ret = taosNetCheckRpc(host, port, sendpkgLen, spi, NULL); - if (ret < 0) { + tsRpcForceTcp = 1; + int32_t ret = taosNetCheckRpc(host, port, sendpkgLen, spi, NULL); + if (ret < 0) { printf("failed to test TCP port:%d\n", port); - } else { + } else { printf("successed to test TCP port:%d\n", port); - } + } - if (pkgLen >= tsRpcMaxUdpSize) { + if (pkgLen >= tsRpcMaxUdpSize) { sendpkgLen = tsRpcMaxUdpSize - 1000; - } else { + } else { sendpkgLen = pkgLen; - } - - tsRpcForceTcp = 0; - ret = taosNetCheckRpc(host, port, pkgLen, spi, NULL); - if (ret < 0) { + } +/* + tsRpcForceTcp = 0; + ret = taosNetCheckRpc(host, port, pkgLen, spi, NULL); + if (ret < 0) { printf("failed to test UDP port:%d\n", port); - } else { + } else { printf("successed to test UDP port:%d\n", port); - } } + */ - taosNetCheckSync(host, startPort + TSDB_PORT_SYNC); - taosNetCheckSync(host, startPort + TSDB_PORT_ARBITRATOR); + taosNetCheckSync(host, startPort); } static void taosNetTestClient(char *host, int32_t startPort, int32_t pkgLen) { - int32_t endPort = startPort + 11; - uInfo("work as client, host:%s startPort:%d endPort:%d pkgLen:%d\n", host, startPort, endPort, pkgLen); + uInfo("work as client, host:%s Port:%d pkgLen:%d\n", host, startPort, pkgLen); uint32_t serverIp = taosGetIpv4FromFqdn(host); if (serverIp == 0xFFFFFFFF) { @@ -508,20 +547,19 @@ static void taosNetTestClient(char *host, int32_t startPort, int32_t pkgLen) { } uInfo("server ip:%s is resolved from host:%s", taosIpStr(serverIp), host); - taosNetCheckPort(serverIp, startPort, endPort, pkgLen); + taosNetCheckPort(serverIp, startPort, startPort, pkgLen); } static void taosNetTestServer(char *host, int32_t startPort, int32_t pkgLen) { - int32_t endPort = startPort + 11; - uInfo("work as server, host:%s startPort:%d endPort:%d pkgLen:%d\n", host, startPort, endPort, pkgLen); + uInfo("work as server, host:%s Port:%d pkgLen:%d\n", host, startPort, pkgLen); int32_t port = startPort; - int32_t num = endPort - startPort + 1; + int32_t num = 1; if (num < 0) num = 1; - TdThread *pids = malloc(2 * num * sizeof(TdThread)); - STestInfo *tinfos = malloc(num * sizeof(STestInfo)); - STestInfo *uinfos = malloc(num * sizeof(STestInfo)); + TdThread *pids = taosMemoryMalloc(2 * num * sizeof(TdThread)); + STestInfo *tinfos = taosMemoryMalloc(num * sizeof(STestInfo)); + STestInfo *uinfos = taosMemoryMalloc(num * sizeof(STestInfo)); for (int32_t i = 0; i < num; i++) { STestInfo *tcpInfo = tinfos + i; @@ -578,7 +616,7 @@ static void taosNetCheckSpeed(char *host, int32_t port, int32_t pkgLen, } tsCompressMsgSize = -1; - SRpcEpSet epSet; + SEpSet epSet; SRpcMsg reqMsg; SRpcMsg rspMsg; void * pRpcConn; @@ -596,11 +634,10 @@ static void taosNetCheckSpeed(char *host, int32_t port, int32_t pkgLen, for (int32_t i = 1; i <= pkgNum; i++) { uint64_t startTime = taosGetTimestampUs(); - memset(&epSet, 0, sizeof(SRpcEpSet)); - epSet.inUse = 0; + memset(&epSet, 0, sizeof(SEpSet)); + strcpy(epSet.eps[0].fqdn, host); + epSet.eps[0].port = port; epSet.numOfEps = 1; - epSet.port[0] = port; - strcpy(epSet.fqdn[0], host); reqMsg.msgType = TDMT_DND_NETWORK_TEST; reqMsg.pCont = rpcMallocCont(pkgLen); @@ -641,7 +678,7 @@ static void taosNetCheckSpeed(char *host, int32_t port, int32_t pkgLen, void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, int32_t pkgNum, char *pkgType) { - tscEmbedded = 1; + tsLogEmbedded = 1; if (host == NULL) host = tsLocalFqdn; if (port == 0) port = tsServerPort; if (0 == strcmp("speed", role)){ @@ -659,14 +696,14 @@ void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, } else if (0 == strcmp("server", role)) { taosNetTestServer(host, port, pkgLen); } else if (0 == strcmp("rpc", role)) { - tscEmbedded = 0; + tsLogEmbedded = 0; taosNetTestRpc(host, port, pkgLen); } else if (0 == strcmp("sync", role)) { taosNetCheckSync(host, port); } else if (0 == strcmp("startup", role)) { taosNetTestStartup(host, port); } else if (0 == strcmp("speed", role)) { - tscEmbedded = 0; + tsLogEmbedded = 0; char type[10] = {0}; taosNetCheckSpeed(host, port, pkgLen, pkgNum, strtolower(type, pkgType)); }else if (0 == strcmp("fqdn", role)) { @@ -675,5 +712,5 @@ void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, taosNetTestStartup(host, port); } - tscEmbedded = 0; + tsLogEmbedded = 0; }