diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 575a0e6274691d3b24429a891979507a36eed543..f87f660ab350eb64ea272ef085e8c14ccfe56eae 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -212,6 +212,11 @@ if(${BUILD_WITH_UV}) MESSAGE("Windows need set no-sign-compare") add_compile_options(-Wno-sign-compare) endif () + if (${CMAKE_SYSTEM_NAME} MATCHES "Windows") + file(READ "libuv/include/uv.h" CONTENTS) + string(REGEX REPLACE "/([\r]*)\nstruct uv_tcp_s {" "/\\1\ntypedef BOOL (PASCAL *LPFN_CONNECTEX) (SOCKET s, const struct sockaddr* name, int namelen, PVOID lpSendBuffer, DWORD dwSendDataLength,LPDWORD lpdwBytesSent, LPOVERLAPPED lpOverlapped);\\1\nstruct uv_tcp_s {" CONTENTS_NEW "${CONTENTS}") + file(WRITE "libuv/include/uv.h" "${CONTENTS_NEW}") + endif () add_subdirectory(libuv) endif(${BUILD_WITH_UV}) @@ -243,6 +248,10 @@ if(${BUILD_WITH_SQLITE}) endif(${BUILD_WITH_SQLITE}) # pthread +if(${BUILD_PTHREAD}) + ADD_DEFINITIONS("-DPTW32_STATIC_LIB") + add_subdirectory(pthread-win32) +endif(${BUILD_PTHREAD}) # ================================================================================================ diff --git a/include/client/taos.h b/include/client/taos.h index 66e3c491c775791d6a67129f0bd402f5aa1aa898..a102629efa425ca71143669f18091eef073fe414 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -31,27 +31,27 @@ typedef void TAOS_SUB; typedef void **TAOS_ROW; // Data type definition -#define TSDB_DATA_TYPE_NULL 0 // 1 bytes -#define TSDB_DATA_TYPE_BOOL 1 // 1 bytes -#define TSDB_DATA_TYPE_TINYINT 2 // 1 byte -#define TSDB_DATA_TYPE_SMALLINT 3 // 2 bytes -#define TSDB_DATA_TYPE_INT 4 // 4 bytes -#define TSDB_DATA_TYPE_BIGINT 5 // 8 bytes -#define TSDB_DATA_TYPE_FLOAT 6 // 4 bytes -#define TSDB_DATA_TYPE_DOUBLE 7 // 8 bytes -#define TSDB_DATA_TYPE_VARCHAR 8 // string, alias for varchar -#define TSDB_DATA_TYPE_TIMESTAMP 9 // 8 bytes -#define TSDB_DATA_TYPE_NCHAR 10 // unicode string -#define TSDB_DATA_TYPE_UTINYINT 11 // 1 byte -#define TSDB_DATA_TYPE_USMALLINT 12 // 2 bytes -#define TSDB_DATA_TYPE_UINT 13 // 4 bytes -#define TSDB_DATA_TYPE_UBIGINT 14 // 8 bytes -#define TSDB_DATA_TYPE_JSON 15 // json string -#define TSDB_DATA_TYPE_VARBINARY 16 // binary -#define TSDB_DATA_TYPE_DECIMAL 17 // decimal -#define TSDB_DATA_TYPE_BLOB 18 // binary +#define TSDB_DATA_TYPE_NULL 0 // 1 bytes +#define TSDB_DATA_TYPE_BOOL 1 // 1 bytes +#define TSDB_DATA_TYPE_TINYINT 2 // 1 byte +#define TSDB_DATA_TYPE_SMALLINT 3 // 2 bytes +#define TSDB_DATA_TYPE_INT 4 // 4 bytes +#define TSDB_DATA_TYPE_BIGINT 5 // 8 bytes +#define TSDB_DATA_TYPE_FLOAT 6 // 4 bytes +#define TSDB_DATA_TYPE_DOUBLE 7 // 8 bytes +#define TSDB_DATA_TYPE_VARCHAR 8 // string, alias for varchar +#define TSDB_DATA_TYPE_TIMESTAMP 9 // 8 bytes +#define TSDB_DATA_TYPE_NCHAR 10 // unicode string +#define TSDB_DATA_TYPE_UTINYINT 11 // 1 byte +#define TSDB_DATA_TYPE_USMALLINT 12 // 2 bytes +#define TSDB_DATA_TYPE_UINT 13 // 4 bytes +#define TSDB_DATA_TYPE_UBIGINT 14 // 8 bytes +#define TSDB_DATA_TYPE_JSON 15 // json string +#define TSDB_DATA_TYPE_VARBINARY 16 // binary +#define TSDB_DATA_TYPE_DECIMAL 17 // decimal +#define TSDB_DATA_TYPE_BLOB 18 // binary #define TSDB_DATA_TYPE_MEDIUMBLOB 19 -#define TSDB_DATA_TYPE_BINARY TSDB_DATA_TYPE_VARCHAR // string +#define TSDB_DATA_TYPE_BINARY TSDB_DATA_TYPE_VARCHAR // string typedef enum { TSDB_OPTION_LOCALE, @@ -257,9 +257,15 @@ DLL_EXPORT void tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_co void tmqShowMsg(tmq_message_t *tmq_message); int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message); -typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code); +/* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */ -DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT* stmt); +DLL_EXPORT TAOS_ROW tmq_get_row(tmq_message_t *message); +DLL_EXPORT char *tmq_get_topic_name(tmq_message_t *message); + +/* ---------------------- OTHER ---------------------------- */ +typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code); + +DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt); #ifdef __cplusplus } diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 3ca120981835cbec20d971aa3ca2e5fbad549dad..385c123fec8f87c7c0785916139d2082e273f33c 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -62,18 +62,17 @@ typedef struct SDataBlockInfo { union {int64_t uid; int64_t blockId;}; } SDataBlockInfo; -typedef struct SConstantItem { - SColumnInfo info; - int32_t startRow; // run-length-encoding to save the space for multiple rows - int32_t endRow; - SVariant value; -} SConstantItem; +//typedef struct SConstantItem { +// SColumnInfo info; +// int32_t startRow; // run-length-encoding to save the space for multiple rows +// int32_t endRow; +// SVariant value; +//} SConstantItem; // info.numOfCols = taosArrayGetSize(pDataBlock) + taosArrayGetSize(pConstantList); typedef struct SSDataBlock { SColumnDataAgg *pBlockAgg; SArray *pDataBlock; // SArray - SArray *pConstantList; // SArray, it is a constant/tags value of the corresponding result value. SDataBlockInfo info; } SSDataBlock; @@ -95,66 +94,15 @@ typedef struct SColumnInfoData { }; } SColumnInfoData; -static FORCE_INLINE int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { - int64_t tbUid = pBlock->info.uid; - int16_t numOfCols = pBlock->info.numOfCols; - int16_t hasVarCol = pBlock->info.hasVarCol; - int32_t rows = pBlock->info.rows; - int32_t sz = taosArrayGetSize(pBlock->pDataBlock); - - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, tbUid); - tlen += taosEncodeFixedI16(buf, numOfCols); - tlen += taosEncodeFixedI16(buf, hasVarCol); - tlen += taosEncodeFixedI32(buf, rows); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - SColumnInfoData* pColData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); - tlen += taosEncodeFixedI16(buf, pColData->info.colId); - tlen += taosEncodeFixedI16(buf, pColData->info.type); - tlen += taosEncodeFixedI32(buf, pColData->info.bytes); - int32_t colSz = rows * pColData->info.bytes; - tlen += taosEncodeBinary(buf, pColData->pData, colSz); - } - return tlen; -} - -static FORCE_INLINE void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) { - int32_t sz; - - buf = taosDecodeFixedI64(buf, &pBlock->info.uid); - buf = taosDecodeFixedI16(buf, &pBlock->info.numOfCols); - buf = taosDecodeFixedI16(buf, &pBlock->info.hasVarCol); - buf = taosDecodeFixedI32(buf, &pBlock->info.rows); - buf = taosDecodeFixedI32(buf, &sz); - pBlock->pDataBlock = taosArrayInit(sz, sizeof(SColumnInfoData)); - for (int32_t i = 0; i < sz; i++) { - SColumnInfoData data = {0}; - buf = taosDecodeFixedI16(buf, &data.info.colId); - buf = taosDecodeFixedI16(buf, &data.info.type); - buf = taosDecodeFixedI32(buf, &data.info.bytes); - int32_t colSz = pBlock->info.rows * data.info.bytes; - buf = taosDecodeBinary(buf, (void**)&data.pData, colSz); - taosArrayPush(pBlock->pDataBlock, &data); - } - return (void*)buf; -} +void* blockDataDestroy(SSDataBlock* pBlock); +int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock); +void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock); static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { if (pBlock == NULL) { return; } - - // int32_t numOfOutput = pBlock->info.numOfCols; - int32_t sz = taosArrayGetSize(pBlock->pDataBlock); - for (int32_t i = 0; i < sz; ++i) { - SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); - tfree(pColInfoData->pData); - } - - taosArrayDestroy(pBlock->pDataBlock); - tfree(pBlock->pBlockAgg); - // tfree(pBlock); + blockDataDestroy(pBlock); } static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp) { diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 7e60013aa1911371b648229cdfd8e6cbfe3c2dc5..17e019d9775fb5714a95490db44937b3706736bc 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -52,6 +52,21 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet); BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \ } while (0) +static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) { + if (!pColumnInfoData->hasNull) { + return false; + } + if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { + return pColumnInfoData->varmeta.offset[row] == -1; + } else { + if (pColumnInfoData->nullbitmap == NULL) { + return false; + } + + return colDataIsNull_f(pColumnInfoData->nullbitmap, row); + } +} + static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, uint32_t totalRows, uint32_t row, SColumnDataAgg* pColAgg) { if (!pColumnInfoData->hasNull) { @@ -79,10 +94,10 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u } } -#define BitmapLen(_n) (((_n) + ((1<> NBIT) - +#define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT) -#define colDataGetData(p1_, r_) \ +// SColumnInfoData, rowNumber +#define colDataGetData(p1_, r_) \ ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? ((p1_)->pData + (p1_)->varmeta.offset[(r_)]) \ : ((p1_)->pData + ((r_) * (p1_)->info.bytes))) @@ -126,4 +141,4 @@ void* blockDataDestroy(SSDataBlock* pBlock); } #endif -#endif /*_TD_COMMON_EP_H_*/ +#endif /*_TD_COMMON_EP_H_*/ diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a837dd97083b6258a64c325809e7507e3dd8a8b4..bb03e96df4220069934eee20a4d74a8fb0b505c2 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -419,7 +419,7 @@ typedef struct { }; } SColumnFilterList; /* - * for client side struct, we only need the column id, type, bytes are not necessary + * for client side struct, only column id, type, bytes are necessary * But for data in vnode side, we need all the following information. */ typedef struct { @@ -2158,13 +2158,36 @@ typedef struct { SArray* topics; // SArray } SMqCMGetSubEpRsp; -struct tmq_message_t { +typedef struct { SMqRspHead head; union { SMqPollRsp consumeRsp; SMqCMGetSubEpRsp getEpRsp; }; void* extra; +} SMqMsgWrapper; + +typedef struct { + int32_t curBlock; + int32_t curRow; + void** uData; +} SMqRowIter; + +struct tmq_message_t_v1 { + SMqPollRsp rsp; + SMqRowIter iter; +}; + +struct tmq_message_t { + SMqRspHead head; + union { + SMqPollRsp consumeRsp; + SMqCMGetSubEpRsp getEpRsp; + }; + void* extra; + int32_t curBlock; + int32_t curRow; + void** uData; }; static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taosArrayDestroy(pSubTopicEp->vgs); } diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 6a07887721bd222338a6a1292229c6fbfa2d11bd..a596794b3d9ac493f775bd2d6f6bb74f12075036 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -189,6 +189,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp) TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqCVConsumeReq, SMqCVConsumeRsp) + TD_DEF_MSG_TYPE(TDMT_VND_TASK_DEPLOY, "vnode-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp) TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 8871bae4e9012e0bb0af1dcf4081879948bdb734..1d89b075395c89e42475cacf94867842dec058f5 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -69,7 +69,7 @@ typedef struct SProjectLogicNode { } SProjectLogicNode; typedef struct SVnodeModifLogicNode { - SLogicNode node;; + SLogicNode node; int32_t msgType; SArray* pDataBlocks; SVgDataBlocks* pVgDataBlocks; @@ -124,7 +124,7 @@ typedef struct SSubLogicPlan { } SSubLogicPlan; typedef struct SQueryLogicPlan { - ENodeType type;; + ENodeType type; int32_t totalLevel; SNodeList* pTopSubplans; } SQueryLogicPlan; @@ -252,7 +252,7 @@ typedef struct SSubplan { } SSubplan; typedef struct SQueryPlan { - ENodeType type;; + ENodeType type; uint64_t queryId; int32_t numOfSubplans; SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0. diff --git a/include/os/os.h b/include/os/os.h index 12f4e733c22a2418af2b4ad80a692ab6c8f7e803..9118a92e11beff1c0403616632be4ff4c5493195 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -25,10 +25,11 @@ extern "C" { #include #include +#include + #if !defined(WINDOWS) #include #include -#include #include #include #include @@ -36,6 +37,12 @@ extern "C" { #include #include #include +#include +#include +#include +#include +#include +#include #if defined(DARWIN) #else @@ -61,12 +68,7 @@ extern "C" { #include #include #include -#include -#include -#include -#include #include -#include #include diff --git a/include/os/osDef.h b/include/os/osDef.h index fcc96c19b46eb51da82ea5fcc0d87b7a4b90d742..492df1047c75ee43010f9a10712d05668d598624 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -56,6 +56,7 @@ extern "C" { // specific typedef int (*__compar_fn_t)(const void *, const void *); #define ssize_t int + #define _SSIZE_T_ #define bzero(ptr, size) memset((ptr), 0, (size)) #define strcasecmp _stricmp #define strncasecmp _strnicmp diff --git a/include/os/osFile.h b/include/os/osFile.h index 58a9df25042638c72d84f79ae57873f87b173249..209cecedf8946ae51da4cebdbd98dd73a017aedf 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -24,7 +24,7 @@ extern "C" { #if defined(WINDOWS) typedef int32_t FileFd; -typedef SOCKET SocketFd; +typedef int32_t SocketFd; #else typedef int32_t FileFd; typedef int32_t SocketFd; diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 520d3af3313131ef7a7e8936a0de963374a3ff26..fb330a9e2fc62cc90f9047ace4488a954a51f50b 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -25,6 +25,8 @@ #define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID #define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID #define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID + #define inet_addr INET_ADDR_FUNC_TAOS_FORBID + #define inet_ntoa INET_NTOA_FUNC_TAOS_FORBID #endif #if defined(WINDOWS) diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 7777466973e414ed4aa12f77e865585023a0fd2f..e1cc6f78299f9638583f277cd99ad94b640661d4 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -16,7 +16,6 @@ #ifndef _TD_OS_SYSINFO_H_ #define _TD_OS_SYSINFO_H_ -#include #include "os.h" #ifdef __cplusplus diff --git a/include/util/tencode.h b/include/util/tencode.h index cbbd55c8a3216e4b580bf724518a11ed368635e6..7e1b624dfb5095536a8c5f170b80af38afc68d52 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -75,7 +75,7 @@ typedef struct { #define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos) #define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE)) #define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE)) -#define TCODER_MALLOC(SIZE, CODER) TFL_MALLOC(SIZE, &((CODER)->fl)) +#define TCODER_MALLOC(PTR, TYPE, SIZE, CODER) TFL_MALLOC(PTR, TYPE, SIZE, &((CODER)->fl)) void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size, td_coder_t type); void tCoderClear(SCoder* pCoder); diff --git a/include/util/tfreelist.h b/include/util/tfreelist.h index c1913ebaf95866218db3d6b151a903f040e2175e..0a507aeec9b35951e7e646559cf058d5cb05b69a 100644 --- a/include/util/tfreelist.h +++ b/include/util/tfreelist.h @@ -29,15 +29,17 @@ struct SFreeListNode { typedef TD_SLIST(SFreeListNode) SFreeList; -#define TFL_MALLOC(SIZE, LIST) \ - ({ \ +#define TFL_MALLOC(PTR, TYPE, SIZE, LIST) \ + do { \ void *ptr = malloc((SIZE) + sizeof(struct SFreeListNode)); \ if (ptr) { \ TD_SLIST_PUSH((LIST), (struct SFreeListNode *)ptr); \ ptr = ((struct SFreeListNode *)ptr)->payload; \ + (PTR) = (TYPE)(ptr); \ + }else{ \ + (PTR) = NULL; \ } \ - ptr; \ - }) + }while(0); #define tFreeListInit(pFL) TD_SLIST_INIT(pFL) diff --git a/include/util/types.h b/include/util/types.h index f7a535c965fd6deac8dc2eb1d486c072a0e867e6..981c457fc10598d7109c44845939364c428f7a66 100644 --- a/include/util/types.h +++ b/include/util/types.h @@ -82,7 +82,7 @@ typedef uint16_t VarDataLenT; // maxVarDataLen: 32767 #define VARSTR_HEADER_SIZE sizeof(VarDataLenT) #define varDataLen(v) ((VarDataLenT *)(v))[0] -#define varDataVal(v) ((void *)((char *)v + VARSTR_HEADER_SIZE)) +#define varDataVal(v) ((char *)(v) + VARSTR_HEADER_SIZE) typedef int32_t VarDataOffsetT; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 0ddd53be0838bf7c633a43b6849470b35f4c8be5..ee13527b968c5d648cbe83e4f57b7e72c71d7eed 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -700,6 +700,10 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { } memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead)); tDecodeSMqPollRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->consumeRsp); + pRsp->curBlock = 0; + pRsp->curRow = 0; + // TODO: alloc mem + /*pRsp->*/ /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/ if (pRsp->consumeRsp.numOfTopics == 0) { /*printf("no data\n");*/ @@ -758,9 +762,9 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { goto END; } - // tmq's epoch is monotomically increase, + // tmq's epoch is monotonically increase, // so it's safe to discard any old epoch msg. - // epoch will only increase when received newer epoch ep msg + // Epoch will only increase when received newer epoch ep msg SMqRspHead* head = pMsg->pData; int32_t epoch = atomic_load_32(&tmq->epoch); if (head->epoch <= epoch) { @@ -1282,6 +1286,34 @@ const char* tmq_err2str(tmq_resp_err_t err) { return "fail"; } +TAOS_ROW tmq_get_row(tmq_message_t* message) { + SMqPollRsp* rsp = &message->consumeRsp; + while (1) { + if (message->curBlock < taosArrayGetSize(rsp->pBlockData)) { + SSDataBlock* pBlock = taosArrayGet(rsp->pBlockData, message->curBlock); + if (message->curRow < pBlock->info.rows) { + for (int i = 0; i < pBlock->info.numOfCols; i++) { + SColumnInfoData* pData = taosArrayGet(pBlock->pDataBlock, i); + if (colDataIsNull_s(pData, message->curRow)) + message->uData[i] = NULL; + else { + message->uData[i] = colDataGetData(pData, message->curRow); + } + } + message->curRow++; + return message->uData; + } else { + message->curBlock++; + message->curRow = 0; + continue; + } + } + return NULL; + } +} + +char* tmq_get_topic_name(tmq_message_t* message) { return "not implemented yet"; } + #if 0 tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) { tmq_t* pTmq = malloc(sizeof(tmq_t)); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 0d1f762db6e226fbbf32df16a4399d281aa82c1e..4b649d5f62e7139929d5aac87e20b3f0914d290f 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -240,10 +240,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co } size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) { - ASSERT(pBlock); - - size_t constantCols = (pBlock->pConstantList != NULL)? taosArrayGetSize(pBlock->pConstantList):0; - ASSERT( pBlock->info.numOfCols == taosArrayGetSize(pBlock->pDataBlock) + constantCols); + ASSERT(pBlock && pBlock->info.numOfCols == taosArrayGetSize(pBlock->pDataBlock)); return pBlock->info.numOfCols; } @@ -1166,3 +1163,67 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock) { size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) { return pageSize / (blockDataGetSerialRowSize(pBlock) + blockDataGetSerialMetaSize(pBlock)); } + +int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { + int64_t tbUid = pBlock->info.uid; + int16_t numOfCols = pBlock->info.numOfCols; + int16_t hasVarCol = pBlock->info.hasVarCol; + int32_t rows = pBlock->info.rows; + int32_t sz = taosArrayGetSize(pBlock->pDataBlock); + + int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, tbUid); + tlen += taosEncodeFixedI16(buf, numOfCols); + tlen += taosEncodeFixedI16(buf, hasVarCol); + tlen += taosEncodeFixedI32(buf, rows); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SColumnInfoData* pColData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); + tlen += taosEncodeFixedI16(buf, pColData->info.colId); + tlen += taosEncodeFixedI16(buf, pColData->info.type); + tlen += taosEncodeFixedI32(buf, pColData->info.bytes); + + if (IS_VAR_DATA_TYPE(pColData->info.type)) { + tlen += taosEncodeBinary(buf, pColData->varmeta.offset, sizeof(int32_t) * rows); + } else { + tlen += taosEncodeBinary(buf, pColData->nullbitmap, BitmapLen(rows)); + } + + int32_t len = colDataGetLength(pColData, rows); + taosEncodeFixedI32(buf, len); + + tlen += taosEncodeBinary(buf, pColData->pData, len); + } + return tlen; +} + +void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) { + int32_t sz; + + buf = taosDecodeFixedI64(buf, &pBlock->info.uid); + buf = taosDecodeFixedI16(buf, &pBlock->info.numOfCols); + buf = taosDecodeFixedI16(buf, &pBlock->info.hasVarCol); + buf = taosDecodeFixedI32(buf, &pBlock->info.rows); + buf = taosDecodeFixedI32(buf, &sz); + pBlock->pDataBlock = taosArrayInit(sz, sizeof(SColumnInfoData)); + for (int32_t i = 0; i < sz; i++) { + SColumnInfoData data = {0}; + buf = taosDecodeFixedI16(buf, &data.info.colId); + buf = taosDecodeFixedI16(buf, &data.info.type); + buf = taosDecodeFixedI32(buf, &data.info.bytes); + + if (IS_VAR_DATA_TYPE(data.info.type)) { + buf = taosDecodeBinary(buf, (void**)&data.varmeta.offset, pBlock->info.rows * sizeof(int32_t)); + data.varmeta.length = pBlock->info.rows * sizeof(int32_t); + data.varmeta.allocLen = data.varmeta.length; + } else { + buf = taosDecodeBinary(buf, (void**)&data.nullbitmap, BitmapLen(pBlock->info.rows)); + } + + int32_t len = 0; + buf = taosDecodeFixedI32(buf, &len); + buf = taosDecodeBinary(buf, (void**)&data.pData, len); + taosArrayPush(pBlock->pDataBlock, &data); + } + return (void*)buf; +} \ No newline at end of file diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 794e5a85426f68c6d564b62aed987bcd90aebc03..d9242cf0ac68b65c264d2c74028f9a513eb7f3c6 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2467,7 +2467,7 @@ int32_t tEncodeSMqCMCommitOffsetReq(SCoder *encoder, const SMqCMCommitOffsetReq int32_t tDecodeSMqCMCommitOffsetReq(SCoder *decoder, SMqCMCommitOffsetReq *pReq) { if (tStartDecode(decoder) < 0) return -1; if (tDecodeI32(decoder, &pReq->num) < 0) return -1; - pReq->offsets = TCODER_MALLOC(pReq->num * sizeof(SMqOffset), decoder); + TCODER_MALLOC(pReq->offsets, SMqOffset*, pReq->num * sizeof(SMqOffset), decoder); if (pReq->offsets == NULL) return -1; for (int32_t i = 0; i < pReq->num; i++) { tDecodeSMqOffset(decoder, &pReq->offsets[i]); diff --git a/source/common/src/tname.c b/source/common/src/tname.c index a97792de663ae25600ae07a9c01f7bea19888eaa..370bf0396999d25a2ef019088a4c90ec3e7e3485 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -217,7 +217,7 @@ int32_t tNameSetDbName(SName* dst, int32_t acct, const char* dbName, size_t name } int32_t tNameSetAcctId(SName* dst, int32_t acctId) { - assert(dst != NULL && acct != NULL); + assert(dst != NULL); dst->acctId = acctId; return 0; } diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index cc5bf843ce22164c051950b4c6e25780fcdc3fe7..f81dead3259c3e43d0287829dfcd33be6ba2fad1 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -27,6 +27,22 @@ int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) { if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1; if (tEncodeCStr(pEncoder, pObj->logicalPlan) < 0) return -1; if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1; + // TODO encode tasks + if (pObj->tasks) { + int32_t sz = taosArrayGetSize(pObj->tasks); + tEncodeI32(pEncoder, sz); + for (int32_t i = 0; i < sz; i++) { + SArray *pArray = taosArrayGet(pObj->tasks, i); + int32_t innerSz = taosArrayGetSize(pArray); + tEncodeI32(pEncoder, innerSz); + for (int32_t j = 0; j < innerSz; j++) { + SStreamTask *pTask = taosArrayGet(pArray, j); + tEncodeSStreamTask(pEncoder, pTask); + } + } + } else { + tEncodeI32(pEncoder, 0); + } return pEncoder->pos; } @@ -42,5 +58,23 @@ int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) { if (tDecodeCStrAlloc(pDecoder, &pObj->sql) < 0) return -1; if (tDecodeCStrAlloc(pDecoder, &pObj->logicalPlan) < 0) return -1; if (tDecodeCStrAlloc(pDecoder, &pObj->physicalPlan) < 0) return -1; + int32_t sz; + if (tDecodeI32(pDecoder, &sz) < 0) return -1; + if (sz != 0) { + pObj->tasks = taosArrayInit(sz, sizeof(SArray)); + for (int32_t i = 0; i < sz; i++) { + int32_t innerSz; + if (tDecodeI32(pDecoder, &innerSz) < 0) return -1; + SArray *pArray = taosArrayInit(innerSz, sizeof(SStreamTask)); + for (int32_t j = 0; j < innerSz; j++) { + SStreamTask task; + if (tDecodeSStreamTask(pDecoder, &task) < 0) return -1; + taosArrayPush(pArray, &task); + } + taosArrayPush(pObj->tasks, pArray); + } + } else { + pObj->tasks = NULL; + } return 0; } diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 16b1ba8a5c9cc86db1735e7a87da81c969eda4cf..faef757e7625b14d69c395284c5e7a5921711c97 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -77,6 +77,32 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { return -1; } taosArrayPush(taskOneLevel, pTask); + + SCoder encoder; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); + tEncodeSStreamTask(&encoder, pTask); + int32_t tlen = sizeof(SMsgHead) + encoder.pos; + tCoderClear(&encoder); + void* buf = rpcMallocCont(tlen); + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + ((SMsgHead*)buf)->streamTaskId = pTask->taskId; + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tCoderInit(&encoder, TD_LITTLE_ENDIAN, abuf, tlen, TD_ENCODER); + tEncodeSStreamTask(&encoder, pTask); + tCoderClear(&encoder); + + STransAction action = {0}; + action.epSet = plan->execNode.epSet; + action.pCont = buf; + action.contLen = tlen; + action.msgType = TDMT_VND_TASK_DEPLOY; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + rpcFreeCont(buf); + return -1; + } } } else if (plan->subplanType == SUBPLAN_TYPE_SCAN) { // duplicatable @@ -101,6 +127,32 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { } taosArrayPush(taskOneLevel, pTask); + + SCoder encoder; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); + tEncodeSStreamTask(&encoder, pTask); + int32_t tlen = sizeof(SMsgHead) + encoder.pos; + tCoderClear(&encoder); + void* buf = rpcMallocCont(tlen); + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + ((SMsgHead*)buf)->streamTaskId = pTask->taskId; + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tCoderInit(&encoder, TD_LITTLE_ENDIAN, abuf, tlen, TD_ENCODER); + tEncodeSStreamTask(&encoder, pTask); + tCoderClear(&encoder); + + STransAction action = {0}; + action.epSet = plan->execNode.epSet; + action.pCont = buf; + action.contLen = tlen; + action.msgType = TDMT_SND_TASK_DEPLOY; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + rpcFreeCont(buf); + return -1; + } } } else { // not duplicatable @@ -117,6 +169,32 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { return -1; } taosArrayPush(taskOneLevel, pTask); + + SCoder encoder; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); + tEncodeSStreamTask(&encoder, pTask); + int32_t tlen = sizeof(SMsgHead) + encoder.pos; + tCoderClear(&encoder); + void* buf = rpcMallocCont(tlen); + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + ((SMsgHead*)buf)->streamTaskId = pTask->taskId; + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tCoderInit(&encoder, TD_LITTLE_ENDIAN, abuf, tlen, TD_ENCODER); + tEncodeSStreamTask(&encoder, pTask); + tCoderClear(&encoder); + + STransAction action = {0}; + action.epSet = plan->execNode.epSet; + action.pCont = buf; + action.contLen = tlen; + action.msgType = TDMT_SND_TASK_DEPLOY; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + rpcFreeCont(buf); + return -1; + } } taosArrayPush(pStream->tasks, taskOneLevel); } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 67011dfe8a3979e315eb234a0223ee245dcb9ea0..af0f354c11dc668353582e9a9cf862165df25e10 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -230,19 +230,19 @@ static int32_t mndCreateStream(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateStreamR } mDebug("trans:%d, used to create stream:%s", pTrans->id, pCreate->name); - SSdbRaw *pRedoRaw = mndStreamActionEncode(&streamObj); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + if (mndScheduleStream(pMnode, pTrans, &streamObj) < 0) { + mError("stream:%ld, schedule stream since %s", streamObj.uid, terrstr()); mndTransDrop(pTrans); return -1; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); - if (mndScheduleStream(pMnode, pTrans, &streamObj) < 0) { - mError("stream:%ld, schedule stream since %s", streamObj.uid, terrstr()); + SSdbRaw *pRedoRaw = mndStreamActionEncode(&streamObj); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index b4d45e83fb340202d39b5674dfbcf1d74b37179f..8d8ed2e427087798c7e6da03be47743cec526bfa 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -55,6 +55,7 @@ int tqCommit(STQ*); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessSetConnReq(STQ* pTq, char* msg); int32_t tqProcessRebReq(STQ* pTq, char* msg); +int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index fc6a3699d555774baa6366aad67b70653bc46411..9b21cf92ce83a161e56031cf8569f1337f3a180c 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -161,6 +161,7 @@ struct STQ { STqMemRef tqMemRef; STqMetaStore* tqMeta; STqPushMgr* tqPushMgr; + SHashObj* pStreamTasks; SWal* pWal; SMeta* pVnodeMeta; }; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index d15481b4aa54c6c5009f0030eca63b0685a749ec..8cdc250e8d5bbb9cbaac311a38ac2001144572a1 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -55,6 +55,8 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pVnodeMeta, STqCfg* tqConfig, S return NULL; } + pTq->pStreamTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + return pTq; } @@ -416,3 +418,18 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { terrno = TSDB_CODE_SUCCESS; return 0; } + +int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) { + SStreamTask* pTask = malloc(sizeof(SStreamTask)); + if (pTask == NULL) { + return -1; + } + SCoder decoder; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, (uint8_t*)msg, msgLen, TD_DECODER); + tDecodeSStreamTask(&decoder, pTask); + tCoderClear(&decoder); + + taosHashPut(pTq->pStreamTasks, &pTask->taskId, sizeof(int32_t), pTask, sizeof(SStreamTask)); + + return 0; +} diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index aba8100478e973bb84deb22beac5038f243f95f2..218b53c2abe42a8d0f197fc11db395339d68f9a5 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -41,7 +41,7 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { return 0; } -int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { +int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { void *ptr = NULL; if (pVnode->config.streamMode == 0) { @@ -63,7 +63,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { switch (pMsg->msgType) { case TDMT_VND_CREATE_STB: { - SVCreateTbReq vCreateTbReq = {0}; + SVCreateTbReq vCreateTbReq = {0}; tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbReq); if (metaCreateTable(pVnode->pMeta, &(vCreateTbReq)) < 0) { // TODO: handle error @@ -100,7 +100,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { break; } case TDMT_VND_ALTER_STB: { - SVCreateTbReq vAlterTbReq = {0}; + SVCreateTbReq vAlterTbReq = {0}; vTrace("vgId:%d, process alter stb req", pVnode->vgId); tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vAlterTbReq); free(vAlterTbReq.stbCfg.pSchema); @@ -132,6 +132,11 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { if (tqProcessRebReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { } } break; + case TDMT_VND_TASK_DEPLOY: { + if (tqProcessTaskDeploy(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), + pMsg->contLen - sizeof(SMsgHead)) < 0) { + } + } break; case TDMT_VND_CREATE_SMA: { // timeRangeSMA SSmaCfg vCreateSmaReq = {0}; if (tDeserializeSVCreateTSmaReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateSmaReq) == NULL) { diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 5a5e1d46c622ec9d854614b9ddb4724bd047e710..1d0cbf22df0e245776ec7b1128a09fb0eb2364fd 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -89,7 +89,7 @@ static SSubLogicPlan* stsCreateScanSubplan(SSplitContext* pCxt, SScanLogicNode* pSubplan->id.groupId = pCxt->groupId; pSubplan->subplanType = SUBPLAN_TYPE_SCAN; pSubplan->pNode = (SLogicNode*)nodesCloneNode(pScan); - TSWAP(pSubplan->pVgroupList, ((SScanLogicNode*)pSubplan->pNode)->pVgroupList, SVgroupsInfo); + TSWAP(pSubplan->pVgroupList, ((SScanLogicNode*)pSubplan->pNode)->pVgroupList, SVgroupsInfo*); SPLIT_FLAG_SET_MASK(pSubplan->splitFlag, SPLIT_FLAG_STS); return pSubplan; } diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index 5785089a20beb96df31dcb7eb6a003d607f68d8e..ffb2ae3a26e3e356185e41ffc606d2672b6af320 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -163,15 +163,15 @@ typedef struct SyncClientRequest { } SyncClientRequest; SyncClientRequest* syncClientRequestBuild(uint32_t dataLen); -SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak); +SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak); // step 1 void syncClientRequestDestroy(SyncClientRequest* pMsg); void syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen); void syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg); char* syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len); SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len); -void syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg); +void syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg); // step 2 void syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg); -SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg); +SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg); // step 3 cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg); char* syncClientRequest2Str(const SyncClientRequest* pMsg); diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index 6ba27b0d8a3812e97d151bbcf10a47a0ce19ed4c..037a49f45c5e7a43f77d2dfff13edf7be56398fd 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -40,12 +40,13 @@ typedef struct SSyncRaftEntry { } SSyncRaftEntry; SSyncRaftEntry* syncEntryBuild(uint32_t dataLen); -SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index); +SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index); // step 4 void syncEntryDestory(SSyncRaftEntry* pEntry); -char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len); -SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len); +char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len); // step 5 +SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len); // step 6 cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry); char* syncEntry2Str(const SSyncRaftEntry* pEntry); +void syncEntry2OriginalRpc(const SSyncRaftEntry* pEntry, SRpcMsg* pRpcMsg); // step 7 // for debug ---------------------- void syncEntryPrint(const SSyncRaftEntry* pObj); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 888d7e16d1152c2a7d9de3718277041e46774eaa..7013d281e3f04a8614663c97ea8990359bac8482 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -101,7 +101,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { assert(pMsg->dataLen >= 0); SyncTerm localPreLogTerm = 0; - if (pMsg->prevLogTerm >= SYNC_INDEX_BEGIN && pMsg->prevLogTerm <= ths->pLogStore->getLastIndex(ths->pLogStore)) { + if (pMsg->prevLogIndex >= SYNC_INDEX_BEGIN && pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { SSyncRaftEntry* pEntry = logStoreGetEntry(ths->pLogStore, pMsg->prevLogIndex); assert(pEntry != NULL); localPreLogTerm = pEntry->term; @@ -174,7 +174,12 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { pReply->destId = pMsg->srcId; pReply->term = ths->pRaftStore->currentTerm; pReply->success = true; - pReply->matchIndex = pMsg->prevLogIndex + 1; + + if (pMsg->dataLen > 0) { + pReply->matchIndex = pMsg->prevLogIndex + 1; + } else { + pReply->matchIndex = pMsg->prevLogIndex; + } SRpcMsg rpcMsg; syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index f9630f7cfe37e4391a9a88568bb76ca53fc301a7..93539db9386dd061fa680a83fc1d2c955f933326 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -39,7 +39,8 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* p syncAppendEntriesReplyLog2("==syncNodeOnAppendEntriesReplyCb==", pMsg); if (pMsg->term < ths->pRaftStore->currentTerm) { - sTrace("DropStaleResponse, receive term:%" PRIu64 ", current term:%" PRIu64 "", pMsg->term, ths->pRaftStore->currentTerm); + sTrace("DropStaleResponse, receive term:%" PRIu64 ", current term:%" PRIu64 "", pMsg->term, + ths->pRaftStore->currentTerm); return ret; } diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 850468f393a6b1ba0e2b3aab6a9dd23eaeaeb4f3..25835edc926a3b5b7f1b176860d5984a47291be7 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -15,6 +15,7 @@ #include "syncIndexMgr.h" #include "syncInt.h" +#include "syncRaftLog.h" // \* Leader i advances its commitIndex. // \* This is done as a separate step from handling AppendEntries responses, @@ -42,4 +43,25 @@ void syncNodeMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { syncIndexMgrLog2("==syncNodeMaybeAdvanceCommitIndex== pNextIndex", pSyncNode->pNextIndex); syncIndexMgrLog2("==syncNodeMaybeAdvanceCommitIndex== pMatchIndex", pSyncNode->pMatchIndex); + + // update commit index + + if (pSyncNode->pFsm != NULL) { + SyncIndex beginIndex = SYNC_INDEX_INVALID; + SyncIndex endIndex = SYNC_INDEX_INVALID; + for (SyncIndex i = beginIndex; i <= endIndex; ++i) { + SSyncRaftEntry* pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, i); + assert(pEntry != NULL); + + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pEntry, &rpcMsg); + + if (pSyncNode->pFsm->FpCommitCb != NULL) { + pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0); + } + + rpcFreeCont(rpcMsg.pCont); + syncEntryDestory(pEntry); + } + } } \ No newline at end of file diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index 4b650ec087024c20f318797399baf20197ae895d..d47a525e2584652a700b2a9d4da2721fa0571dc2 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -54,20 +54,22 @@ static void syncEnvTick(void *param, void *tmrId) { SSyncEnv *pSyncEnv = (SSyncEnv *)param; if (atomic_load_64(&pSyncEnv->envTickTimerLogicClockUser) <= atomic_load_64(&pSyncEnv->envTickTimerLogicClock)) { ++(pSyncEnv->envTickTimerCounter); - sTrace( - "syncEnvTick do ... envTickTimerLogicClockUser:%" PRIu64 ", envTickTimerLogicClock:%" PRIu64 ", envTickTimerCounter:%" PRIu64 ", " - "envTickTimerMS:%d, tmrId:%p", - pSyncEnv->envTickTimerLogicClockUser, pSyncEnv->envTickTimerLogicClock, pSyncEnv->envTickTimerCounter, - pSyncEnv->envTickTimerMS, tmrId); + sTrace("syncEnvTick do ... envTickTimerLogicClockUser:%" PRIu64 ", envTickTimerLogicClock:%" PRIu64 + ", envTickTimerCounter:%" PRIu64 + ", " + "envTickTimerMS:%d, tmrId:%p", + pSyncEnv->envTickTimerLogicClockUser, pSyncEnv->envTickTimerLogicClock, pSyncEnv->envTickTimerCounter, + pSyncEnv->envTickTimerMS, tmrId); // do something, tick ... taosTmrReset(syncEnvTick, pSyncEnv->envTickTimerMS, pSyncEnv, pSyncEnv->pTimerManager, &pSyncEnv->pEnvTickTimer); } else { - sTrace( - "syncEnvTick pass ... envTickTimerLogicClockUser:%" PRIu64 ", envTickTimerLogicClock:%" PRIu64 ", envTickTimerCounter:%" PRIu64 ", " - "envTickTimerMS:%d, tmrId:%p", - pSyncEnv->envTickTimerLogicClockUser, pSyncEnv->envTickTimerLogicClock, pSyncEnv->envTickTimerCounter, - pSyncEnv->envTickTimerMS, tmrId); + sTrace("syncEnvTick pass ... envTickTimerLogicClockUser:%" PRIu64 ", envTickTimerLogicClock:%" PRIu64 + ", envTickTimerCounter:%" PRIu64 + ", " + "envTickTimerMS:%d, tmrId:%p", + pSyncEnv->envTickTimerLogicClockUser, pSyncEnv->envTickTimerLogicClock, pSyncEnv->envTickTimerCounter, + pSyncEnv->envTickTimerMS, tmrId); } } diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index c8448c32eb08d3733e9b8e82d6f57ec395d99829..c841c2ac068957bcb82613647cc3ec95a9c7ac2e 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -29,7 +29,7 @@ static int32_t syncIODestroy(SSyncIO *io); static int32_t syncIOStartInternal(SSyncIO *io); static int32_t syncIOStopInternal(SSyncIO *io); -static void *syncIOConsumerFunc(void *param); +static void * syncIOConsumerFunc(void *param); static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); @@ -234,9 +234,9 @@ static int32_t syncIOStopInternal(SSyncIO *io) { } static void *syncIOConsumerFunc(void *param) { - SSyncIO *io = param; + SSyncIO * io = param; STaosQall *qall; - SRpcMsg *pRpcMsg, rpcMsg; + SRpcMsg * pRpcMsg, rpcMsg; qall = taosAllocateQall(); while (1) { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index bd1b4213c013ebf5cdd8ca69165c29267c8c50e4..1f3e709a27e7001e71f064b75d718dfa8f0b0bc9 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -731,14 +731,44 @@ static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg int32_t ret = 0; syncClientRequestLog2("==syncNodeOnClientRequestCb==", pMsg); + SyncIndex index = ths->pLogStore->getLastIndex(ths->pLogStore) + 1; + SyncTerm term = ths->pRaftStore->currentTerm; + SSyncRaftEntry* pEntry = syncEntryBuild2((SyncClientRequest*)pMsg, term, index); + assert(pEntry != NULL); + if (ths->state == TAOS_SYNC_STATE_LEADER) { - SSyncRaftEntry* pEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); ths->pLogStore->appendEntry(ths->pLogStore, pEntry); + + // only myself, maybe commit + syncNodeMaybeAdvanceCommitIndex(ths); + + // start replicate right now! syncNodeReplicate(ths); - syncEntryDestory(pEntry); + + // pre commit + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pEntry, &rpcMsg); + + if (ths->pFsm != NULL) { + if (ths->pFsm->FpPreCommitCb != NULL) { + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0); + } + } + rpcFreeCont(rpcMsg.pCont); + } else { - // ths->pFsm->FpCommitCb(-1) + // pre commit + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pEntry, &rpcMsg); + + if (ths->pFsm != NULL) { + if (ths->pFsm->FpPreCommitCb != NULL) { + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, -1); + } + } + rpcFreeCont(rpcMsg.pCont); } + syncEntryDestory(pEntry); return ret; } diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 453ac13ce338f44448ae6f9154b20d22c66a4fec..3ac4a7a1c083485fe776a92a59c2989b111966a7 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -76,7 +76,7 @@ cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) { free(s); } else { - pRoot = syncRpcUnknownMsg2Json(); + pRoot = cJSON_CreateObject(); char* s; s = syncUtilprintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen); cJSON_AddStringToObject(pRoot, "pCont", s); @@ -608,6 +608,7 @@ SyncClientRequest* syncClientRequestBuild(uint32_t dataLen) { return pMsg; } +// step 1. original SRpcMsg => SyncClientRequest, add seqNum, isWeak SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak) { SyncClientRequest* pMsg = syncClientRequestBuild(pOriginalRpcMsg->contLen); pMsg->originalRpcType = pOriginalRpcMsg->msgType; @@ -652,6 +653,7 @@ SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len) return pMsg; } +// step 2. SyncClientRequest => RpcMsg, to queue void syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg) { memset(pRpcMsg, 0, sizeof(*pRpcMsg)); pRpcMsg->msgType = pMsg->msgType; @@ -664,6 +666,7 @@ void syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg syncClientRequestDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); } +// step 3. RpcMsg => SyncClientRequest, from queue SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg) { SyncClientRequest* pMsg = syncClientRequestDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); return pMsg; diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index 2589c3a5e7aed0ea70c219a8a23771c38d20dd45..e4c5882e9896d423709e2fcc4c8de6245ad908b4 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -26,6 +26,7 @@ SSyncRaftEntry* syncEntryBuild(uint32_t dataLen) { return pEntry; } +// step 4. SyncClientRequest => SSyncRaftEntry, add term, index SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index) { SSyncRaftEntry* pEntry = syncEntryBuild(pMsg->dataLen); assert(pEntry != NULL); @@ -48,6 +49,7 @@ void syncEntryDestory(SSyncRaftEntry* pEntry) { } } +// step 5. SSyncRaftEntry => bin, to raft log char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len) { char* buf = malloc(pEntry->bytes); assert(buf != NULL); @@ -58,6 +60,7 @@ char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len) { return buf; } +// step 6. bin => SSyncRaftEntry, from raft log SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); SSyncRaftEntry* pEntry = malloc(bytes); @@ -106,6 +109,15 @@ char* syncEntry2Str(const SSyncRaftEntry* pEntry) { return serialized; } +// step 7. SSyncRaftEntry => original SRpcMsg, commit to user, delete seqNum, isWeak, term, index +void syncEntry2OriginalRpc(const SSyncRaftEntry* pEntry, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); + pRpcMsg->msgType = pEntry->originalRpcType; + pRpcMsg->contLen = pEntry->dataLen; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + memcpy(pRpcMsg->pCont, pEntry->data, pRpcMsg->contLen); +} + // for debug ---------------------- void syncEntryPrint(const SSyncRaftEntry* pObj) { char* serialized = syncEntry2Str(pObj); diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index ca04db81b6f636ed3eab0726e248deaac1919b14..0bb701fc09a112234622a3f32830ba12e47445b7 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -73,7 +73,8 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { SyncAppendEntries* pMsg = NULL; SSyncRaftEntry* pEntry = logStoreGetEntry(pSyncNode->pLogStore, nextIndex); if (pEntry != NULL) { - SyncAppendEntries* pMsg = syncAppendEntriesBuild(pEntry->bytes); + pMsg = syncAppendEntriesBuild(pEntry->bytes); + assert(pMsg != NULL); // add pEntry into msg uint32_t len; @@ -86,9 +87,11 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { } else { // maybe overflow, send empty record - SyncAppendEntries* pMsg = syncAppendEntriesBuild(0); + pMsg = syncAppendEntriesBuild(0); + assert(pMsg != NULL); } + assert(pMsg != NULL); pMsg->srcId = pSyncNode->myRaftId; pMsg->destId = *pDestId; pMsg->term = pSyncNode->pRaftStore->currentTerm; diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c index 11537c1e98f4513d26b36b3dc65ea51739257e7e..12603bb337ce6791599bd6755756ff3ef2e090a7 100644 --- a/source/libs/sync/src/syncRequestVoteReply.c +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -41,7 +41,8 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) syncRequestVoteReplyLog2("==syncNodeOnRequestVoteReplyCb==", pMsg); if (pMsg->term < ths->pRaftStore->currentTerm) { - sTrace("DropStaleResponse, receive term:%" PRIu64 ", current term:%" PRIu64 "", pMsg->term, ths->pRaftStore->currentTerm); + sTrace("DropStaleResponse, receive term:%" PRIu64 ", current term:%" PRIu64 "", pMsg->term, + ths->pRaftStore->currentTerm); return ret; } diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 90889a7af0e5585a808ea34afe4ff365e1b2341b..9486405331bd3d5352eb8b1ce73cbff950b3feaa 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -14,15 +14,12 @@ */ #include "syncUtil.h" -#include -#include -#include #include "syncEnv.h" // ---- encode / decode uint64_t syncUtilAddr2U64(const char* host, uint16_t port) { uint64_t u64; - uint32_t hostU32 = (uint32_t)inet_addr(host); + uint32_t hostU32 = (uint32_t)taosInetAddr(host); // assert(hostU32 != (uint32_t)-1); u64 = (((uint64_t)hostU32) << 32) | (((uint32_t)port) << 16); return u64; @@ -33,7 +30,7 @@ void syncUtilU642Addr(uint64_t u64, char* host, size_t len, uint16_t* port) { struct in_addr addr; addr.s_addr = hostU32; - snprintf(host, len, "%s", inet_ntoa(addr)); + snprintf(host, len, "%s", taosInetNtoa(addr)); *port = (uint16_t)((u64 & 0x00000000FFFF0000) >> 16); } diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 2583deb32f6c4b3e25a426d29e6f46c821731b34..5fa0e975d88bb043708b58ae37f2514d699492a4 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -28,6 +28,7 @@ add_executable(syncRpcMsgTest "") add_executable(syncPingTimerTest2 "") add_executable(syncPingSelfTest "") add_executable(syncElectTest "") +add_executable(syncEncodeTest "") target_sources(syncTest @@ -150,6 +151,10 @@ target_sources(syncElectTest PRIVATE "syncElectTest.cpp" ) +target_sources(syncEncodeTest + PRIVATE + "syncEncodeTest.cpp" +) target_include_directories(syncTest @@ -307,6 +312,11 @@ target_include_directories(syncElectTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncEncodeTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -429,6 +439,10 @@ target_link_libraries(syncElectTest sync gtest_main ) +target_link_libraries(syncEncodeTest + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncElectTest.cpp b/source/libs/sync/test/syncElectTest.cpp index b8a1460f3555460c58f98b126b5672f38d80ff5a..502263cfbf736758298961211db6eda5876de2a3 100644 --- a/source/libs/sync/test/syncElectTest.cpp +++ b/source/libs/sync/test/syncElectTest.cpp @@ -116,7 +116,9 @@ int main(int argc, char** argv) { //--------------------------- while (1) { - sTrace("while 1 sleep, state: %d, %s", gSyncNode->state, syncUtilState2String(gSyncNode->state)); + sTrace("while 1 sleep, state: %d, %s, electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d", + gSyncNode->state, syncUtilState2String(gSyncNode->state), gSyncNode->electTimerLogicClock, + gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS); taosMsleep(1000); } diff --git a/source/libs/sync/test/syncEncodeTest.cpp b/source/libs/sync/test/syncEncodeTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f615c1d237a75f6f37877ae363597a47e6ef48ea --- /dev/null +++ b/source/libs/sync/test/syncEncodeTest.cpp @@ -0,0 +1,236 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncRaftEntry.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "syncRaftLog.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 1; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; +SWal* pWal; +SSyncNode* pSyncNode; + +SSyncNode* syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./"); + + int code = walInit(); + assert(code == 0); + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = syncInfo.vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + pWal = walOpen("./wal_test", &walCfg); + assert(pWal != NULL); + + syncInfo.pWal = pWal; + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->pSyncNode = pSyncNode; + + return pSyncNode; +} + +SSyncNode* syncInitTest() { return syncNodeInit(); } + +void logStoreTest() { + logStorePrint2((char*)"logStoreTest", pSyncNode->pLogStore); + + assert(pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) == SYNC_INDEX_INVALID); + + for (int i = 0; i < 5; ++i) { + int32_t dataLen = 10; + SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); + assert(pEntry != NULL); + pEntry->msgType = 1; + pEntry->originalRpcType = 2; + pEntry->seqNum = 3; + pEntry->isWeak = true; + pEntry->term = 100; + pEntry->index = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) + 1; + snprintf(pEntry->data, dataLen, "value%d", i); + + // syncEntryPrint2((char*)"write entry:", pEntry); + pSyncNode->pLogStore->appendEntry(pSyncNode->pLogStore, pEntry); + syncEntryDestory(pEntry); + + if (i == 0) { + assert(pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) == SYNC_INDEX_BEGIN); + } + } + logStorePrint2((char*)"after appendEntry", pSyncNode->pLogStore); + + pSyncNode->pLogStore->truncate(pSyncNode->pLogStore, 3); + logStorePrint2((char*)"after truncate 3", pSyncNode->pLogStore); +} + +void initRaftId(SSyncNode* pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } +} + +SRpcMsg *step0() { + SRpcMsg *pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + memset(pMsg, 0, sizeof(SRpcMsg)); + pMsg->msgType = 9999; + pMsg->contLen = 32; + pMsg->pCont = malloc(pMsg->contLen); + snprintf((char *)(pMsg->pCont), pMsg->contLen, "hello, world"); + return pMsg; +} + +SyncClientRequest *step1(const SRpcMsg *pMsg) { + SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true); + return pRetMsg; +} + +SRpcMsg *step2(const SyncClientRequest *pMsg) { + SRpcMsg *pRetMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + syncClientRequest2RpcMsg(pMsg, pRetMsg); + return pRetMsg; +} + +SyncClientRequest *step3(const SRpcMsg *pMsg) { + SyncClientRequest *pRetMsg = syncClientRequestFromRpcMsg2(pMsg); + return pRetMsg; +} + +SSyncRaftEntry *step4(const SyncClientRequest *pMsg) { + SSyncRaftEntry *pRetMsg = syncEntryBuild2((SyncClientRequest *)pMsg, 100, 0); + return pRetMsg; +} + +char *step5(const SSyncRaftEntry *pMsg, uint32_t *len) { + char *pRetMsg = syncEntrySerialize(pMsg, len); + return pRetMsg; +} + +SSyncRaftEntry *step6(const char *pMsg, uint32_t len) { + SSyncRaftEntry *pRetMsg = syncEntryDeserialize(pMsg, len); + return pRetMsg; +} + +SRpcMsg *step7(const SSyncRaftEntry *pMsg) { + SRpcMsg *pRetMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + syncEntry2OriginalRpc(pMsg, pRetMsg); + return pRetMsg; +} + +int main(int argc, char **argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + void logTest(); + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + taosRemoveDir("./wal_test"); + + // step0 + SRpcMsg *pMsg0 = step0(); + syncRpcMsgPrint2((char *)"==step0==", pMsg0); + + // step1 + SyncClientRequest *pMsg1 = step1(pMsg0); + syncClientRequestPrint2((char *)"==step1==", pMsg1); + + // step2 + SRpcMsg *pMsg2 = step2(pMsg1); + syncRpcMsgPrint2((char *)"==step2==", pMsg2); + + // step3 + SyncClientRequest *pMsg3 = step3(pMsg2); + syncClientRequestPrint2((char *)"==step3==", pMsg3); + + // step4 + SSyncRaftEntry *pMsg4 = step4(pMsg3); + syncEntryPrint2((char *)"==step4==", pMsg4); + + // log, relog + SSyncNode* pSyncNode = syncNodeInit(); + assert(pSyncNode != NULL); + SSyncRaftEntry *pEntry = pMsg4; + pSyncNode->pLogStore->appendEntry(pSyncNode->pLogStore, pEntry); + SSyncRaftEntry *pEntry2 = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, pEntry->index); + syncEntryPrint2((char *)"==pEntry2==", pEntry2); + + // step5 + uint32_t len; + char * pMsg5 = step5(pMsg4, &len); + char * s = syncUtilprintBin(pMsg5, len); + printf("==step5== [%s] \n", s); + free(s); + + // step6 + SSyncRaftEntry *pMsg6 = step6(pMsg5, len); + syncEntryPrint2((char *)"==step6==", pMsg6); + + // step7 + SRpcMsg *pMsg7 = step7(pMsg6); + syncRpcMsgPrint2((char *)"==step7==", pMsg7); + + return 0; +} diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 86e7980733f39baa393cc45d87cedf755593900e..a2520babe5575bdbc157a91f7b20c48c4c9ba0de 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -27,7 +27,8 @@ typedef struct { } SBtIdx; // Btree page header definition -typedef struct __attribute__((__packed__)) { +#pragma pack (push,1) +typedef struct { uint8_t flag; // page flag int32_t vlen; // value length of current page, TDB_VARIANT_LEN for variant length uint16_t nPayloads; // number of total payloads @@ -36,6 +37,7 @@ typedef struct __attribute__((__packed__)) { pgoff_t offPayload; // payload offset pgno_t rChildPgno; // right most child page number } SBtPgHdr; +#pragma pack(pop) typedef int (*BtreeCmprFn)(const void *, const void *); diff --git a/source/libs/tdb/src/db/tdbEnv.c b/source/libs/tdb/src/db/tdbEnv.c index a415d6d60e500cae4389aaf79fdc18f60cae4349..30e8fab03902e39c9f2c1c88756518693bf2baf6 100644 --- a/source/libs/tdb/src/db/tdbEnv.c +++ b/source/libs/tdb/src/db/tdbEnv.c @@ -30,11 +30,7 @@ struct STDbEnv { } pgfht; // page file hash table; }; -#define TDB_ENV_PGF_HASH(fileid) \ - ({ \ - uint8_t *tmp = (uint8_t *)(fileid); \ - tmp[0] + tmp[1] + tmp[2]; \ - }) +#define TDB_ENV_PGF_HASH(fileid) (((uint8_t *)(fileid))[0] + ((uint8_t *)(fileid))[1] + ((uint8_t *)(fileid))[2]) static int tdbEnvDestroy(TENV *pEnv); diff --git a/source/libs/tdb/src/db/tdbPgCache.c b/source/libs/tdb/src/db/tdbPgCache.c index 1b000aa6d6f85dd983b0d60a3c2565990cfd815e..ef3f79410a8e03d1b0fa94eb1a555fdb7baac397 100644 --- a/source/libs/tdb/src/db/tdbPgCache.c +++ b/source/libs/tdb/src/db/tdbPgCache.c @@ -106,11 +106,7 @@ int pgCacheClose(SPgCache *pPgCache) { return 0; } -#define PG_CACHE_HASH(fileid, pgno) \ - ({ \ - uint64_t *tmp = (uint64_t *)(fileid); \ - (tmp[0] + tmp[1] + tmp[2] + (pgno)); \ - }) +#define PG_CACHE_HASH(fileid, pgno) (((uint64_t *)(fileid))[0] + ((uint64_t *)(fileid))[1] + ((uint64_t *)(fileid))[2] + (pgno)) SPage *pgCacheFetch(SPgCache *pPgCache, pgid_t pgid) { SPage * pPage; diff --git a/source/libs/tdb/src/inc/tdbPgFile.h b/source/libs/tdb/src/inc/tdbPgFile.h index eaeebf6b9d3d6260b181b0e8fd9607a2befe7482..0a8c277479e6b49e6a1bbf1854ca8c7f6f509306 100644 --- a/source/libs/tdb/src/inc/tdbPgFile.h +++ b/source/libs/tdb/src/inc/tdbPgFile.h @@ -20,13 +20,15 @@ extern "C" { #endif -typedef struct __attribute__((__packed__)) { +#pragma pack (push,1) +typedef struct { char hdrInfo[16]; // info string pgsz_t szPage; // page size of current file int32_t cno; // commit number counter pgno_t freePgno; // freelist page number uint8_t resv[100]; // reserved space } SPgFileHdr; +#pragma pack(pop) #define TDB_PG_FILE_HDR_SIZE 128 diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 31097a591fe8bc97b0b98775cc1753c28d3c9b3e..df35b8d67b2e2b0c91c154c4ff3d7cb951d0d8b1 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -207,8 +207,8 @@ void cliHandleResp(SCliConn* conn) { } tDebug("%s cli conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pTransInst->label, conn, - TMSG_INFO(pHead->msgType), inet_ntoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), - inet_ntoa(conn->locaddr.sin_addr), ntohs(conn->locaddr.sin_port), transMsg.contLen); + TMSG_INFO(pHead->msgType), taosInetNtoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), + taosInetNtoa(conn->locaddr.sin_addr), ntohs(conn->locaddr.sin_port), transMsg.contLen); conn->secured = pHead->secured; @@ -519,8 +519,8 @@ void cliSend(SCliConn* pConn) { uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); tDebug("%s cli conn %p %s is send to %s:%d, local info %s:%d", CONN_GET_INST_LABEL(pConn), pConn, - TMSG_INFO(pHead->msgType), inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), - inet_ntoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); + TMSG_INFO(pHead->msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), + taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); pConn->writeReq.data = pConn; uv_write(&pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb); diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 2efdb109aae693ddd074de8a02de4d8477e1ca6a..73d836319fcabd96fc42790b391fb39d8a6a154d 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -202,7 +202,7 @@ static void uvHandleReq(SSrvConn* pConn) { STransMsgHead* pHead = (STransMsgHead*)p->msg; if (pHead->secured == 1) { - STransUserMsg* uMsg = (p->msg + p->msgLen - sizeof(STransUserMsg)); + STransUserMsg* uMsg = (STransUserMsg*)((char*)p->msg + p->msgLen - sizeof(STransUserMsg)); memcpy(pConn->user, uMsg->user, tListLen(uMsg->user)); memcpy(pConn->secret, uMsg->secret, tListLen(uMsg->secret)); } @@ -235,12 +235,12 @@ static void uvHandleReq(SSrvConn* pConn) { transRefSrvHandle(pConn); transMsg.handle = pConn; tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pConn, TMSG_INFO(transMsg.msgType), - inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), + taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port), transMsg.contLen); } else { tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, no resp ", pConn, - TMSG_INFO(transMsg.msgType), inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), - inet_ntoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port), transMsg.contLen); + TMSG_INFO(transMsg.msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), + taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port), transMsg.contLen); } STrans* pTransInst = (STrans*)p->shandle; @@ -347,7 +347,7 @@ static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { // impl later } tDebug("server conn %p %s is sent to %s:%d, local info: %s:%d", pConn, TMSG_INFO(pHead->msgType), - inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), + taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); pHead->msgLen = htonl(len); @@ -374,8 +374,8 @@ static void uvStartSendResp(SSrvMsg* smsg) { transUnrefSrvHandle(pConn); if (taosArrayGetSize(pConn->srvMsgs) > 0) { - tDebug("server conn %p push data to client %s:%d, local info: %s:%d", pConn, inet_ntoa(pConn->addr.sin_addr), - ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); + tDebug("server conn %p push data to client %s:%d, local info: %s:%d", pConn, taosInetNtoa(pConn->addr.sin_addr), + ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); taosArrayPush(pConn->srvMsgs, &smsg); return; } diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 248f758787d277f0b319d6e89f09a1fde925b80f..70ec7e06557d75358f2b4e33a8aec06b493e06d2 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -19,9 +19,6 @@ #include "tref.h" #include "walInt.h" -#include -#include - int64_t inline walGetFirstVer(SWal* pWal) { return pWal->vers.firstVer; } int64_t inline walGetSnaphostVer(SWal* pWal) { return pWal->vers.snapshotVer; } diff --git a/source/os/CMakeLists.txt b/source/os/CMakeLists.txt index 27906c262753f2bf02bd9cd09b4ca9227ba26bba..ed240015d4dcbff2ddb0c44cbeee7fc751f95429 100644 --- a/source/os/CMakeLists.txt +++ b/source/os/CMakeLists.txt @@ -3,9 +3,10 @@ add_library(os STATIC ${OS_SRC}) target_include_directories( os PUBLIC "${CMAKE_SOURCE_DIR}/include/os" - PRIVATE "${CMAKE_SOURCE_DIR}/include" - PRIVATE "${CMAKE_SOURCE_DIR}/include/util" - PRIVATE "${CMAKE_SOURCE_DIR}/contrib/pthread-win32" + PUBLIC "${CMAKE_SOURCE_DIR}/include" + PUBLIC "${CMAKE_SOURCE_DIR}/include/util" + PUBLIC "${CMAKE_SOURCE_DIR}/contrib/pthread-win32" + PUBLIC "${CMAKE_SOURCE_DIR}/contrib/gnuregex" ) target_link_libraries( os pthread dl rt m diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 062d144933136891acc98b4ba07b43d09c8a9f69..5e3cf1247c6941064a336cb14c5f51c2c6e4013e 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -427,7 +427,7 @@ int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight) { wchar_t *pattern = calloc(varDataLen(pRight) + 1, sizeof(wchar_t)); memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); - int32_t ret = WCSPatternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft) / TSDB_NCHAR_SIZE, &pInfo); + int32_t ret = WCSPatternMatch(pattern, (const wchar_t *)varDataVal(pLeft), varDataLen(pLeft) / TSDB_NCHAR_SIZE, &pInfo); free(pattern); return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; diff --git a/source/util/test/encodeTest.cpp b/source/util/test/encodeTest.cpp index 25314842fb4d417b29dca127a1da70d55b87bd4f..b4da43dfb72600dbcc6ea5fda1fdde220e99c803 100644 --- a/source/util/test/encodeTest.cpp +++ b/source/util/test/encodeTest.cpp @@ -230,7 +230,7 @@ static int32_t tSStructA_v1_decode(SCoder *pCoder, SStructA_v1 *pSAV1) { const char *tstr; uint64_t len; if (tDecodeCStrAndLen(pCoder, &tstr, &len) < 0) return -1; - pSAV1->A_c = (char *)TCODER_MALLOC(len + 1, pCoder); + TCODER_MALLOC(pSAV1->A_c, char*, len + 1, pCoder); memcpy(pSAV1->A_c, tstr, len + 1); tEndDecode(pCoder); @@ -269,7 +269,7 @@ static int32_t tSStructA_v2_decode(SCoder *pCoder, SStructA_v2 *pSAV2) { const char *tstr; uint64_t len; if (tDecodeCStrAndLen(pCoder, &tstr, &len) < 0) return -1; - pSAV2->A_c = (char *)TCODER_MALLOC(len + 1, pCoder); + TCODER_MALLOC(pSAV2->A_c, char*, len + 1, pCoder); memcpy(pSAV2->A_c, tstr, len + 1); // ------------------------NEW FIELDS DECODE------------------------------- @@ -305,7 +305,7 @@ static int32_t tSFinalReq_v1_encode(SCoder *pCoder, const SFinalReq_v1 *ps1) { static int32_t tSFinalReq_v1_decode(SCoder *pCoder, SFinalReq_v1 *ps1) { if (tStartDecode(pCoder) < 0) return -1; - ps1->pA = (SStructA_v1 *)TCODER_MALLOC(sizeof(*(ps1->pA)), pCoder); + TCODER_MALLOC(ps1->pA, SStructA_v1*, sizeof(*(ps1->pA)), pCoder); if (tSStructA_v1_decode(pCoder, ps1->pA) < 0) return -1; if (tDecodeI32(pCoder, &ps1->v_a) < 0) return -1; if (tDecodeI8(pCoder, &ps1->v_b) < 0) return -1; @@ -339,7 +339,7 @@ static int32_t tSFinalReq_v2_encode(SCoder *pCoder, const SFinalReq_v2 *ps2) { static int32_t tSFinalReq_v2_decode(SCoder *pCoder, SFinalReq_v2 *ps2) { if (tStartDecode(pCoder) < 0) return -1; - ps2->pA = (SStructA_v2 *)TCODER_MALLOC(sizeof(*(ps2->pA)), pCoder); + TCODER_MALLOC(ps2->pA, SStructA_v2*, sizeof(*(ps2->pA)), pCoder); if (tSStructA_v2_decode(pCoder, ps2->pA) < 0) return -1; if (tDecodeI32(pCoder, &ps2->v_a) < 0) return -1; if (tDecodeI8(pCoder, &ps2->v_b) < 0) return -1; diff --git a/source/util/test/freelistTest.cpp b/source/util/test/freelistTest.cpp index 7fd3d693fb6ccd3cae2a0a916b3c0aaa7dc6d071..440e997042a4b7f3395f32d5b0eccfe2960a5c0f 100644 --- a/source/util/test/freelistTest.cpp +++ b/source/util/test/freelistTest.cpp @@ -8,7 +8,8 @@ TEST(TD_UTIL_FREELIST_TEST, simple_test) { tFreeListInit(&fl); for (size_t i = 0; i < 1000; i++) { - void *ptr = TFL_MALLOC(1024, &fl); + void *ptr = NULL; + TFL_MALLOC(ptr, void*, 1024, &fl); GTEST_ASSERT_NE(ptr, nullptr); }