diff --git a/cmake/bdb_CMakeLists.txt.in b/cmake/bdb_CMakeLists.txt.in.bak similarity index 100% rename from cmake/bdb_CMakeLists.txt.in rename to cmake/bdb_CMakeLists.txt.in.bak diff --git a/cmake/cmake.options b/cmake/cmake.options index 946eb5d2583096dab9faecc2e8b222c5c24377f5..b51f096185a7d42802af2078fcdefd04d35940bc 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -55,17 +55,6 @@ option( OFF ) -IF(${TD_WINDOWS}) - MESSAGE("Not build BDB on Windows") -ELSE () - option( - BUILD_WITH_BDB - "If build with BerkleyDB" - ON - ) - -ENDIF () - option( BUILD_WITH_LUCENE "If build with lucene" diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 575a0e6274691d3b24429a891979507a36eed543..377e3fbcccc2b0f45bff92eec9e1526b71b1d5ef 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -63,9 +63,9 @@ if(${BUILD_WITH_UV}) endif(${BUILD_WITH_UV}) # bdb -if(${BUILD_WITH_BDB}) - cat("${CMAKE_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) -endif(${BUILD_WITH_BDB}) +#if(${BUILD_WITH_BDB}) + #cat("${CMAKE_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +#endif(${BUILD_WITH_BDB}) # sqlite if(${BUILD_WITH_SQLITE}) diff --git a/contrib/test/CMakeLists.txt b/contrib/test/CMakeLists.txt index eacaeb9524be5dde7a231cfd7090f8dfe45f61ae..740488b39b1efa7de141196983aa6a46dda9f9a9 100644 --- a/contrib/test/CMakeLists.txt +++ b/contrib/test/CMakeLists.txt @@ -7,9 +7,9 @@ if(${BUILD_WITH_LUCENE}) add_subdirectory(lucene) endif(${BUILD_WITH_LUCENE}) -if(${BUILD_WITH_BDB}) - add_subdirectory(bdb) -endif(${BUILD_WITH_BDB}) +#if(${BUILD_WITH_BDB}) + #add_subdirectory(bdb) +#endif(${BUILD_WITH_BDB}) if(${BUILD_WITH_SQLITE}) add_subdirectory(sqlite) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index f2fec58d8fca43336c6ca41e69bd708aa2c76670..bf6a804e7f0f3dae1569c80e064446c2d17d7fae 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -71,7 +71,7 @@ typedef struct SDataBlockInfo { int64_t uid; int64_t blockId; }; - int64_t groupId; // no need to serialize + uint64_t groupId; // no need to serialize } SDataBlockInfo; typedef struct SSDataBlock { diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 797843002751928990f1eb7046ee99a4a3baa5b2..98edd9f0a5c73ec64bba789e1bb6e633f42e39af 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -182,6 +182,7 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd int32_t pageSize); int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock); int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf); +int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity); SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t rowCount); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 3fd5f31c24a72f60063ecdd657cdf290231c405a..9b087c604e23cebc0f937b1c151398731b2aeeb6 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2328,6 +2328,7 @@ typedef struct { char cgroup[TSDB_CGROUP_LEN]; int64_t currentOffset; + uint64_t reqId; char topic[TSDB_TOPIC_FNAME_LEN]; } SMqPollReq; diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 28e54835520f3f4e30c98761d281b8310ced0bbb..39448ceef36d1b64ef0dba831c77a61263cbdcc4 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -30,11 +30,19 @@ extern "C" { #define FOREACH(node, list) \ for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext) -// only be use in FOREACH -#define ERASE_NODE(list) cell = nodesListErase(list, cell); - #define REPLACE_NODE(newNode) cell->pNode = (SNode*)(newNode) +#define INSERT_LIST(target, src) nodesListInsertList((target), cell, src) + +#define WHERE_EACH(node, list) \ + SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); \ + while (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)) + +#define WHERE_NEXT cell = cell->pNext + +// only be use in WHERE_EACH +#define ERASE_NODE(list) cell = nodesListErase((list), cell) + #define FORBOTH(node1, list1, node2, list2) \ for (SListCell* cell1 = (NULL != (list1) ? (list1)->pHead : NULL), *cell2 = (NULL != (list2) ? (list2)->pHead : NULL); \ (NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)), (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), (node1 != NULL && node2 != NULL); \ @@ -202,7 +210,9 @@ int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode); int32_t nodesListMakeAppend(SNodeList** pList, SNodeptr pNode); int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc); int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc); +int32_t nodesListPushFront(SNodeList* pList, SNodeptr pNode); SListCell* nodesListErase(SNodeList* pList, SListCell* pCell); +void nodesListInsertList(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc); SNodeptr nodesListGetNode(SNodeList* pList, int32_t index); void nodesDestroyList(SNodeList* pList); // Only clear the linked list structure, without releasing the elements inside diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index f987afd878f4a6e276b8031508bc5a58f92d3c75..6b6d11c2fef7b89ed16cfb5c4aef569a99b76af3 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -124,6 +124,7 @@ typedef struct STableNode { char dbName[TSDB_DB_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN]; char tableAlias[TSDB_TABLE_NAME_LEN]; + uint8_t precision; } STableNode; struct STableMeta; @@ -236,6 +237,7 @@ typedef struct SSelectStmt { SNode* pLimit; SNode* pSlimit; char stmtName[TSDB_TABLE_NAME_LEN]; + uint8_t precision; } SSelectStmt; typedef enum ESetOperatorType { diff --git a/include/util/taoserror.h b/include/util/taoserror.h index ec9812fa040bc2312d34776ac0d8210776d3df3a..5b7aa51881301117d2e87b4ff549cd4426efdcbf 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -484,16 +484,17 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME TAOS_DEF_ERROR_CODE(0, 0x2617) #define TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR TAOS_DEF_ERROR_CODE(0, 0x2618) #define TSDB_CODE_PAR_INVALID_RANGE_OPTION TAOS_DEF_ERROR_CODE(0, 0x2619) -#define TSDB_CODE_PAR_INVALID_STR_OPTION TAOS_DEF_ERROR_CODE(0, 0x2620) -#define TSDB_CODE_PAR_INVALID_ENUM_OPTION TAOS_DEF_ERROR_CODE(0, 0x2621) -#define TSDB_CODE_PAR_INVALID_TTL_OPTION TAOS_DEF_ERROR_CODE(0, 0x2622) -#define TSDB_CODE_PAR_INVALID_KEEP_NUM TAOS_DEF_ERROR_CODE(0, 0x2623) -#define TSDB_CODE_PAR_INVALID_KEEP_ORDER TAOS_DEF_ERROR_CODE(0, 0x2624) -#define TSDB_CODE_PAR_INVALID_KEEP_VALUE TAOS_DEF_ERROR_CODE(0, 0x2625) -#define TSDB_CODE_PAR_INVALID_COMMENT_OPTION TAOS_DEF_ERROR_CODE(0, 0x2626) -#define TSDB_CODE_PAR_INVALID_F_RANGE_OPTION TAOS_DEF_ERROR_CODE(0, 0x2627) -#define TSDB_CODE_PAR_INVALID_ROLLUP_OPTION TAOS_DEF_ERROR_CODE(0, 0x2628) -#define TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION TAOS_DEF_ERROR_CODE(0, 0x2629) +#define TSDB_CODE_PAR_INVALID_STR_OPTION TAOS_DEF_ERROR_CODE(0, 0x261A) +#define TSDB_CODE_PAR_INVALID_ENUM_OPTION TAOS_DEF_ERROR_CODE(0, 0x261B) +#define TSDB_CODE_PAR_INVALID_TTL_OPTION TAOS_DEF_ERROR_CODE(0, 0x261C) +#define TSDB_CODE_PAR_INVALID_KEEP_NUM TAOS_DEF_ERROR_CODE(0, 0x261D) +#define TSDB_CODE_PAR_INVALID_KEEP_ORDER TAOS_DEF_ERROR_CODE(0, 0x261E) +#define TSDB_CODE_PAR_INVALID_KEEP_VALUE TAOS_DEF_ERROR_CODE(0, 0x261F) +#define TSDB_CODE_PAR_INVALID_COMMENT_OPTION TAOS_DEF_ERROR_CODE(0, 0x2620) +#define TSDB_CODE_PAR_INVALID_F_RANGE_OPTION TAOS_DEF_ERROR_CODE(0, 0x2621) +#define TSDB_CODE_PAR_INVALID_ROLLUP_OPTION TAOS_DEF_ERROR_CODE(0, 0x2622) +#define TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION TAOS_DEF_ERROR_CODE(0, 0x2623) +#define TSDB_CODE_PAR_GROUPBY_WINDOW_COEXIST TAOS_DEF_ERROR_CODE(0, 0x2624) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index a6e5fee2d17fc5fdab7e2f1f80e98e4b650dab4c..5893a14bd57e2b688b9c50c620f8623cb216da7c 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -1149,6 +1149,7 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTo pReq->consumerId = tmq->consumerId; pReq->epoch = tmq->epoch; pReq->currentOffset = reqOffset; + pReq->reqId = generateRequestId(); pReq->head.vgId = htonl(pVg->vgId); pReq->head.contLen = htonl(sizeof(SMqPollReq)); @@ -1279,7 +1280,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { .len = sizeof(SMqPollReq), .handle = NULL, }; - sendInfo->requestId = generateRequestId(); + sendInfo->requestId = pReq->reqId; sendInfo->requestObjRefId = 0; sendInfo->param = pParam; sendInfo->fp = tmqPollCb; @@ -1288,7 +1289,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { int64_t transporterId = 0; /*printf("send poll\n");*/ atomic_add_fetch_32(&tmq->waitingRequest, 1); - tscDebug("consumer %ld send poll: vg %d, epoch %d, req offset %ld", tmq->consumerId, pVg->vgId, tmq->epoch, pVg->currentOffset); + tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %ld, reqId %lu", tmq->consumerId, pTopic->topicName, pVg->vgId, tmq->epoch, pVg->currentOffset, pReq->reqId); /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/ asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); pVg->pollCnt++; diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index fb6640e59eaf129b701249cb87220695c60f6fd9..8f55488e2c75de796c690626e16a25ad3c42b179 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -415,10 +415,7 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = TARRAY_GET_ELEM(pBlock->pDataBlock, i); if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - bool isNull = colDataIsNull(pColInfoData, numOfRows, j, NULL); - if (isNull) { - // do nothing - } else { + if (pColInfoData->varmeta.offset[j] != -1) { char* p = colDataGetData(pColInfoData, j); size += varDataTLen(p); } @@ -547,8 +544,8 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); - size_t metaSize = pBlock->info.rows * sizeof(int32_t); if (IS_VAR_DATA_TYPE(pCol->info.type)) { + size_t metaSize = pBlock->info.rows * sizeof(int32_t); memcpy(pCol->varmeta.offset, pStart, metaSize); pStart += metaSize; } else { @@ -581,6 +578,49 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { return TSDB_CODE_SUCCESS; } +int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity) { + pBlock->info.rows = *(int32_t*)buf; + + int32_t numOfCols = pBlock->info.numOfCols; + const char* pStart = buf + sizeof(uint32_t); + + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); + + if (IS_VAR_DATA_TYPE(pCol->info.type)) { + size_t metaSize = capacity * sizeof(int32_t); + memcpy(pCol->varmeta.offset, pStart, metaSize); + pStart += metaSize; + } else { + memcpy(pCol->nullbitmap, pStart, BitmapLen(capacity)); + pStart += BitmapLen(capacity); + } + + int32_t colLength = *(int32_t*)pStart; + pStart += sizeof(int32_t); + + if (IS_VAR_DATA_TYPE(pCol->info.type)) { + if (pCol->varmeta.allocLen < colLength) { + char* tmp = taosMemoryRealloc(pCol->pData, colLength); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pCol->pData = tmp; + pCol->varmeta.allocLen = colLength; + } + + pCol->varmeta.length = colLength; + ASSERT(pCol->varmeta.length <= pCol->varmeta.allocLen); + } + + memcpy(pCol->pData, pStart, colLength); + pStart += pCol->info.bytes * capacity; + } + + return TSDB_CODE_SUCCESS; +} + size_t blockDataGetRowSize(SSDataBlock* pBlock) { ASSERT(pBlock != NULL); if (pBlock->info.rowSize == 0) { @@ -627,6 +667,10 @@ double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { return rowSize; } +int32_t getAllowedRowsForPage(const SSDataBlock* pBlock, size_t pgSize) { + return (int32_t) ((pgSize - blockDataGetSerialMetaSize(pBlock))/ blockDataGetSerialRowSize(pBlock)); +} + typedef struct SSDataBlockSortHelper { SArray* orderInfo; // SArray SSDataBlock* pDataBlock; @@ -1178,6 +1222,9 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { int32_t code = 0; + if (numOfRows == 0) { + return TSDB_CODE_SUCCESS; + } for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); @@ -1220,7 +1267,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock) { } size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) { - return pageSize / (blockDataGetSerialRowSize(pBlock) + blockDataGetSerialMetaSize(pBlock)); + return (int32_t) ((pageSize - blockDataGetSerialMetaSize(pBlock))/ blockDataGetSerialRowSize(pBlock)); } void colDataDestroy(SColumnInfoData* pColData) { diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 4ecacb9d9d5cf95f81f09d0ace146555a0cca7cf..b351536839673021d4faffffd09695e2cbf69d2f 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -394,7 +394,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, 0) != 0) return -1; tsNumOfVnodeQueryThreads = tsNumOfCores / 2; - tsNumOfVnodeQueryThreads = TMIN(tsNumOfVnodeQueryThreads, 1); + tsNumOfVnodeQueryThreads = TMAX(tsNumOfVnodeQueryThreads, 1); if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 1, 1024, 0) != 0) return -1; tsNumOfVnodeFetchThreads = tsNumOfCores / 2; @@ -402,11 +402,11 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 1, 1024, 0) != 0) return -1; tsNumOfVnodeWriteThreads = tsNumOfCores; - tsNumOfVnodeWriteThreads = TMIN(tsNumOfVnodeWriteThreads, 1); + tsNumOfVnodeWriteThreads = TMAX(tsNumOfVnodeWriteThreads, 1); if (cfgAddInt32(pCfg, "numOfVnodeWriteThreads", tsNumOfVnodeWriteThreads, 1, 1024, 0) != 0) return -1; tsNumOfVnodeSyncThreads = tsNumOfCores / 2; - tsNumOfVnodeSyncThreads = TMIN(tsNumOfVnodeSyncThreads, 1); + tsNumOfVnodeSyncThreads = TMAX(tsNumOfVnodeSyncThreads, 1); if (cfgAddInt32(pCfg, "numOfVnodeSyncThreads", tsNumOfVnodeSyncThreads, 1, 1024, 0) != 0) return -1; tsNumOfVnodeMergeThreads = tsNumOfCores / 8; @@ -414,7 +414,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "numOfVnodeMergeThreads", tsNumOfVnodeMergeThreads, 1, 1024, 0) != 0) return -1; tsNumOfQnodeQueryThreads = tsNumOfCores / 2; - tsNumOfQnodeQueryThreads = TMIN(tsNumOfQnodeQueryThreads, 1); + tsNumOfQnodeQueryThreads = TMAX(tsNumOfQnodeQueryThreads, 1); if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 1, 1024, 0) != 0) return -1; tsNumOfQnodeFetchThreads = tsNumOfCores / 2; diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index e5e591b4d2cee9f0e22e694348fc3c3f89f5346b..965ed792e7bb1644ec16e88fcbf70cc734e9d1d3 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -416,8 +416,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SNodeMsg *pReq, SMCreateSmaReq *pCre streamObj.version = 1; streamObj.sql = pCreate->sql; streamObj.createdBy = STREAM_CREATED_BY__SMA; - // TODO - streamObj.fixedSinkVgId = 0; + streamObj.fixedSinkVgId = smaObj.dstVgId; streamObj.smaId = smaObj.uid; /*streamObj.physicalPlan = "";*/ streamObj.logicalPlan = "not implemented"; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 8078338238ec2f9b0f4a53722034f6db365b28e5..dbd5e43b6df3e8aa85a0b6153c7f6c6898deaba1 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -60,7 +60,7 @@ static int32_t mndProcessResetOffsetReq(SNodeMsg *pMsg); static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup, const SMqConsumerEp *pConsumerEp); -static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp); +static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp, const char* topicName); static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp, const char* oldTopicName); int32_t mndInitSubscribe(SMnode *pMnode) { @@ -102,12 +102,13 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj return pSub; } -static int32_t mndBuildRebalanceMsg(void **pBuf, int32_t *pLen, const SMqConsumerEp *pConsumerEp) { +static int32_t mndBuildRebalanceMsg(void **pBuf, int32_t *pLen, const SMqConsumerEp *pConsumerEp, const char* topicName) { SMqMVRebReq req = { .vgId = pConsumerEp->vgId, .oldConsumerId = pConsumerEp->oldConsumerId, .newConsumerId = pConsumerEp->consumerId, }; + req.topic = strdup(topicName); int32_t tlen = tEncodeSMqMVRebReq(NULL, &req); void *buf = taosMemoryMalloc(sizeof(SMsgHead) + tlen); @@ -122,6 +123,7 @@ static int32_t mndBuildRebalanceMsg(void **pBuf, int32_t *pLen, const SMqConsume void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); tEncodeSMqMVRebReq(&abuf, &req); + taosMemoryFree(req.topic); *pBuf = buf; *pLen = tlen; @@ -129,12 +131,12 @@ static int32_t mndBuildRebalanceMsg(void **pBuf, int32_t *pLen, const SMqConsume return 0; } -static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp) { +static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp, const char* topicName) { ASSERT(pConsumerEp->oldConsumerId != -1); void *buf; int32_t tlen; - if (mndBuildRebalanceMsg(&buf, &tlen, pConsumerEp) < 0) { + if (mndBuildRebalanceMsg(&buf, &tlen, pConsumerEp, topicName) < 0) { return -1; } @@ -502,10 +504,10 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { pConsumerEp->epoch = 0; taosArrayPush(pSubConsumer->vgInfo, pConsumerEp); + char topic[TSDB_TOPIC_FNAME_LEN]; + char cgroup[TSDB_CGROUP_LEN]; + mndSplitSubscribeKey(pSub->key, topic, cgroup); if (pConsumerEp->oldConsumerId == -1) { - char topic[TSDB_TOPIC_FNAME_LEN]; - char cgroup[TSDB_CGROUP_LEN]; - mndSplitSubscribeKey(pSub->key, topic, cgroup); SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); mInfo("mq set conn: assign vgroup %d of topic %s to consumer %" PRId64 " cgroup: %s", pConsumerEp->vgId, @@ -517,7 +519,7 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { mInfo("mq rebalance: assign vgroup %d, from consumer %" PRId64 " to consumer %" PRId64 "", pConsumerEp->vgId, pConsumerEp->oldConsumerId, pConsumerEp->consumerId); - mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp); + mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp, topic); } } } @@ -849,7 +851,7 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) { pConsumerEp->consumerId); mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp); } else { - mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp); + mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp, newTopicName); } // to trigger rebalance at once, do not set status active /*atomic_store_32(&pConsumer->status, MQ_CONSUMER_STATUS__ACTIVE);*/ diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 6b21f935c4ef675e678428ba8948fa1dd030d5bf..b5fe7d460f6dc2ea8406dc4de2e2cc69badaaffb 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -1,54 +1,76 @@ -set(META_DB_IMPL_LIST "BDB" "TDB") -set(META_DB_IMPL "TDB" CACHE STRING "Use BDB as the default META implementation") -set_property(CACHE META_DB_IMPL PROPERTY STRINGS ${META_DB_IMPL_LIST}) - -if(META_DB_IMPL IN_LIST META_DB_IMPL_LIST) - message(STATUS "META DB Impl: ${META_DB_IMPL}==============") -else() - message(FATAL_ERROR "Invalid META DB IMPL: ${META_DB_IMPL}==============") -endif() +# vnode +add_library(vnode STATIC "") +target_sources( + vnode + PRIVATE + # vnode + "src/vnd/vnodeArenaMAImpl.c" + "src/vnd/vnodeBufferPool.c" + "src/vnd/vnodeCfg.c" + "src/vnd/vnodeCommit.c" + "src/vnd/vnodeInt.c" + "src/vnd/vnodeMain.c" + "src/vnd/vnodeMgr.c" + "src/vnd/vnodeQuery.c" + "src/vnd/vnodeStateMgr.c" + "src/vnd/vnodeWrite.c" -aux_source_directory(src/meta META_SRC) -if(${META_DB_IMPL} STREQUAL "BDB") - list(REMOVE_ITEM META_SRC "src/meta/metaTDBImpl.c") -elseif(${META_DB_IMPL} STREQUAL "TDB") - list(REMOVE_ITEM META_SRC "src/meta/metaBDBImpl.c") -endif() + # meta + # "src/meta/metaBDBImpl.c" + "src/meta/metaCache.c" + "src/meta/metaCfg.c" + "src/meta/metaIdx.c" + "src/meta/metaMain.c" + "src/meta/metaQuery.c" + "src/meta/metaTable.c" + "src/meta/metaTbCfg.c" + "src/meta/metaTbTag.c" + "src/meta/metaTbUid.c" + "src/meta/metaTDBImpl.c" -aux_source_directory(src/tq TQ_SRC) -aux_source_directory(src/tsdb TSDB_SRC) -aux_source_directory(src/vnd VND_SRC) + # tsdb + # "src/tsdb/tsdbBDBImpl.c" + "src/tsdb/tsdbCommit.c" + "src/tsdb/tsdbCompact.c" + "src/tsdb/tsdbFile.c" + "src/tsdb/tsdbFS.c" + "src/tsdb/tsdbMain.c" + "src/tsdb/tsdbMemTable.c" + "src/tsdb/tsdbOptions.c" + "src/tsdb/tsdbRead.c" + "src/tsdb/tsdbReadImpl.c" + "src/tsdb/tsdbScan.c" + # "src/tsdb/tsdbSma.c" + "src/tsdb/tsdbWrite.c" -list(APPEND - VNODE_SRC - ${META_SRC} - ${TQ_SRC} - ${TSDB_SRC} - ${VND_SRC} + # tq + "src/tq/tq.c" + "src/tq/tqCommit.c" + "src/tq/tqMetaStore.c" + "src/tq/tqOffset.c" + "src/tq/tqPush.c" + "src/tq/tqRead.c" ) - -add_library(vnode STATIC ${VNODE_SRC}) target_include_directories( vnode - PUBLIC inc - PRIVATE src/inc + PUBLIC "inc" + PRIVATE "src/inc" ) target_link_libraries( vnode PUBLIC os PUBLIC util PUBLIC common - PUBLIC transport PUBLIC tfs PUBLIC wal - PUBLIC scheduler - PUBLIC executor - PUBLIC stream PUBLIC qworker PUBLIC sync - # TODO: get rid of BDB - PUBLIC bdb + PUBLIC executor + PUBLIC scheduler PUBLIC tdb + #PUBLIC bdb + PUBLIC transport + PUBLIC stream ) if(${BUILD_TEST}) diff --git a/source/dnode/vnode/inc/meta.h b/source/dnode/vnode/inc/meta.h deleted file mode 100644 index 149aac1206344f0526edcfe0f7451525ea96845e..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/inc/meta.h +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_META_H_ -#define _TD_META_H_ - -#include "tmallocator.h" -#include "tmsg.h" -#include "trow.h" - -#ifdef __cplusplus -extern "C" { -#endif - -#define META_SUPER_TABLE TD_SUPER_TABLE -#define META_CHILD_TABLE TD_CHILD_TABLE -#define META_NORMAL_TABLE TD_NORMAL_TABLE - -// Types exported -typedef struct SMeta SMeta; - -typedef struct SMetaCfg { - /// LRU cache size - uint64_t lruSize; -} SMetaCfg; - -typedef struct SMTbCursor SMTbCursor; -typedef struct SMCtbCursor SMCtbCursor; -typedef struct SMSmaCursor SMSmaCursor; - -typedef SVCreateTbReq STbCfg; -typedef SVCreateTSmaReq SSmaCfg; - -// SMeta operations -SMeta * metaOpen(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF); -void metaClose(SMeta *pMeta); -void metaRemove(const char *path); -int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg); -int metaDropTable(SMeta *pMeta, tb_uid_t uid); -int metaCommit(SMeta *pMeta); -int32_t metaCreateTSma(SMeta *pMeta, SSmaCfg *pCfg); -int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid); - -// For Query -STbCfg * metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid); -STbCfg * metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid); -SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline); -STSchema * metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver); -STSma * metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid); -STSmaWrapper * metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid); -SArray * metaGetSmaTbUids(SMeta *pMeta, bool isDup); -int metaGetTbNum(SMeta *pMeta); - -SMTbCursor *metaOpenTbCursor(SMeta *pMeta); -void metaCloseTbCursor(SMTbCursor *pTbCur); -char * metaTbCursorNext(SMTbCursor *pTbCur); - -SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid); -void metaCloseCtbCurosr(SMCtbCursor *pCtbCur); -tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur); - -SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid); -void metaCloseSmaCurosr(SMSmaCursor *pSmaCur); -const char * metaSmaCursorNext(SMSmaCursor *pSmaCur); - -// Options -void metaOptionsInit(SMetaCfg *pMetaCfg); -void metaOptionsClear(SMetaCfg *pMetaCfg); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_META_H_*/ diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h deleted file mode 100644 index 9e391a1b7c32dffca19737284bfef1c37def8cb1..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/inc/tsdb.h +++ /dev/null @@ -1,288 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TSDB_H_ -#define _TD_TSDB_H_ - -#include "tmallocator.h" -#include "meta.h" -#include "tcommon.h" -#include "tfs.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct SDataStatis { - int16_t colId; - int16_t maxIndex; - int16_t minIndex; - int16_t numOfNull; - int64_t sum; - int64_t max; - int64_t min; -} SDataStatis; - -typedef struct STable { - uint64_t tid; - uint64_t uid; - STSchema *pSchema; -} STable; - -#define BLOCK_LOAD_OFFSET_SEQ_ORDER 1 -#define BLOCK_LOAD_TABLE_SEQ_ORDER 2 -#define BLOCK_LOAD_TABLE_RR_ORDER 3 - -#define TABLE_TID(t) (t)->tid -#define TABLE_UID(t) (t)->uid - -// TYPES EXPOSED -typedef struct STsdb STsdb; - -typedef struct STsdbCfg { - int8_t precision; - int8_t update; - int8_t compression; - int32_t daysPerFile; - int32_t minRowsPerFileBlock; - int32_t maxRowsPerFileBlock; - int32_t keep; - int32_t keep1; - int32_t keep2; - uint64_t lruCacheSize; - SArray *retentions; -} STsdbCfg; - -// query condition to build multi-table data block iterator -typedef struct STsdbQueryCond { - STimeWindow twindow; - int32_t order; // desc|asc order to iterate the data block - int32_t numOfCols; - SColumnInfo *colList; - bool loadExternalRows; // load external rows or not - int32_t type; // data block load type: -} STsdbQueryCond; - -typedef struct { - TSKEY lastKey; - uint64_t uid; -} STableKeyInfo; - -// STsdb -STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta, STfs *pTfs); -void tsdbClose(STsdb *); -void tsdbRemove(const char *path); -int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp); -int tsdbPrepareCommit(STsdb *pTsdb); -int tsdbCommit(STsdb *pTsdb); - - -int32_t tsdbInitSma(STsdb *pTsdb); -int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg); -int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg); -/** - * @brief When submit msg received, update the relative expired window synchronously. - * - * @param pTsdb - * @param msg - * @return int32_t - */ -int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, SSubmitReq *pMsg); - -/** - * @brief Insert tSma(Time-range-wise SMA) data from stream computing engine - * - * @param pTsdb - * @param indexUid - * @param msg - * @return int32_t - */ -int32_t tsdbInsertTSmaData(STsdb *pTsdb, int64_t indexUid, const char *msg); - -/** - * @brief Drop tSma data and local cache. - * - * @param pTsdb - * @param indexUid - * @return int32_t - */ -int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid); - -/** - * @brief Insert RSma(Rollup SMA) data. - * - * @param pTsdb - * @param msg - * @return int32_t - */ -int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg); - -// TODO: This is the basic params, and should wrap the params to a queryHandle. -/** - * @brief Get tSma(Time-range-wise SMA) data. - * - * @param pTsdb - * @param pData - * @param indexUid - * @param querySKey - * @param nMaxResult - * @return int32_t - */ -int32_t tsdbGetTSmaData(STsdb *pTsdb, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult); - -// STsdbCfg -int tsdbOptionsInit(STsdbCfg *); -void tsdbOptionsClear(STsdbCfg *); - -typedef void* tsdbReaderT; - -/** - * Get the data block iterator, starting from position according to the query condition - * - * @param tsdb tsdb handle - * @param pCond query condition, including time window, result set order, and basic required columns for each block - * @param tableInfoGroup table object list in the form of set, grouped into different sets according to the - * group by condition - * @param qinfo query info handle from query processor - * @return - */ -tsdbReaderT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, uint64_t taskId); - -/** - * Get the last row of the given query time window for all the tables in STableGroupInfo object. - * Note that only one data block with only row will be returned while invoking retrieve data block function for - * all tables in this group. - * - * @param tsdb tsdb handle - * @param pCond query condition, including time window, result set order, and basic required columns for each block - * @param tableInfo table list. - * @return - */ -//tsdbReaderT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, uint64_t qId, -// SMemRef *pRef); - - -tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, void* pMemRef); - -int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* pReader, STableBlockDistInfo* pTableBlockInfo); - -bool isTsdbCacheLastRow(tsdbReaderT* pReader); - -/** - * - * @param tsdb - * @param uid - * @param skey - * @param pTagCond - * @param len - * @param tagNameRelType - * @param tbnameCond - * @param pGroupInfo - * @param pColIndex - * @param numOfCols - * @param reqId - * @return - */ -int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, - int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo, - SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId, uint64_t taskId); -/** - * get num of rows in mem table - * - * @param pHandle - * @return row size - */ - -int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT* pHandle); - -/** - * move to next block if exists - * - * @param pTsdbReadHandle - * @return - */ -bool tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle); - -/** - * Get current data block information - * - * @param pTsdbReadHandle - * @param pBlockInfo - * @return - */ -void tsdbRetrieveDataBlockInfo(tsdbReaderT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo); - -/** - * - * Get the pre-calculated information w.r.t. current data block. - * - * In case of data block in cache, the pBlockStatis will always be NULL. - * If a block is not completed loaded from disk, the pBlockStatis will be NULL. - - * @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0 - * @return - */ -int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT *pTsdbReadHandle, SDataStatis **pBlockStatis); - -/** - * - * The query condition with primary timestamp is passed to iterator during its constructor function, - * the returned data block must be satisfied with the time window condition in any cases, - * which means the SData data block is not actually the completed disk data blocks. - * - * @param pTsdbReadHandle query handle - * @param pColumnIdList required data columns id list - * @return - */ -SArray *tsdbRetrieveDataBlock(tsdbReaderT *pTsdbReadHandle, SArray *pColumnIdList); - -/** - * destroy the created table group list, which is generated by tag query - * @param pGroupList - */ -void tsdbDestroyTableGroup(STableGroupInfo *pGroupList); - -/** - * create the table group result including only one table, used to handle the normal table query - * - * @param tsdb tsdbHandle - * @param uid table uid - * @param pGroupInfo the generated result - * @return - */ -int32_t tsdbGetOneTableGroup(void *pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo); - -/** - * - * @param tsdb - * @param pTableIdList - * @param pGroupInfo - * @return - */ -int32_t tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo); - -/** - * clean up the query handle - * @param queryHandle - */ -void tsdbCleanupReadHandle(tsdbReaderT queryHandle); - -int32_t tdScanAndConvertSubmitMsg(SSubmitReq *pMsg); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TSDB_H_*/ diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index c1c000295a6de4734a8abd7bc9a74aea47ee5e01..113501a26b41fba6846a3299e13a946189e36188 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -21,12 +21,17 @@ #include "tqueue.h" #include "trpc.h" -#include "meta.h" #include "tarray.h" #include "tfs.h" -#include "tsdb.h" #include "wal.h" +#include "tmallocator.h" +#include "tmsg.h" +#include "trow.h" +#include "tmallocator.h" +#include "tcommon.h" +#include "tfs.h" + #ifdef __cplusplus extern "C" { #endif @@ -34,6 +39,82 @@ extern "C" { /* ------------------------ TYPES EXPOSED ------------------------ */ typedef struct SMgmtWrapper SMgmtWrapper; typedef struct SVnode SVnode; + +#define META_SUPER_TABLE TD_SUPER_TABLE +#define META_CHILD_TABLE TD_CHILD_TABLE +#define META_NORMAL_TABLE TD_NORMAL_TABLE + +// Types exported +typedef struct SMeta SMeta; + +typedef struct SMetaCfg { + /// LRU cache size + uint64_t lruSize; +} SMetaCfg; + +typedef struct SMTbCursor SMTbCursor; +typedef struct SMCtbCursor SMCtbCursor; +typedef struct SMSmaCursor SMSmaCursor; + +typedef SVCreateTbReq STbCfg; +typedef SVCreateTSmaReq SSmaCfg; + +typedef struct SDataStatis { + int16_t colId; + int16_t maxIndex; + int16_t minIndex; + int16_t numOfNull; + int64_t sum; + int64_t max; + int64_t min; +} SDataStatis; + +typedef struct STsdbQueryCond { + STimeWindow twindow; + int32_t order; // desc|asc order to iterate the data block + int32_t numOfCols; + SColumnInfo *colList; + bool loadExternalRows; // load external rows or not + int32_t type; // data block load type: +} STsdbQueryCond; + +typedef struct { + TSKEY lastKey; + uint64_t uid; +} STableKeyInfo; + + +typedef struct STable { + uint64_t tid; + uint64_t uid; + STSchema *pSchema; +} STable; + +#define BLOCK_LOAD_OFFSET_SEQ_ORDER 1 +#define BLOCK_LOAD_TABLE_SEQ_ORDER 2 +#define BLOCK_LOAD_TABLE_RR_ORDER 3 + +#define TABLE_TID(t) (t)->tid +#define TABLE_UID(t) (t)->uid + +// TYPES EXPOSED +typedef struct STsdb STsdb; + +typedef struct STsdbCfg { + int8_t precision; + int8_t update; + int8_t compression; + int32_t daysPerFile; + int32_t minRowsPerFileBlock; + int32_t maxRowsPerFileBlock; + int32_t keep; + int32_t keep1; + int32_t keep2; + uint64_t lruCacheSize; + SArray *retentions; +} STsdbCfg; + + typedef struct { // TODO int32_t reserved; @@ -263,6 +344,241 @@ int tqRetrieveDataBlockInfo(STqReadHandle *pHandle, SDataBlockInfo *pBlockIn // return SArray SArray *tqRetrieveDataBlock(STqReadHandle *pHandle); +// meta.h +SMeta *metaOpen(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF); +void metaClose(SMeta *pMeta); +void metaRemove(const char *path); +int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg); +int metaDropTable(SMeta *pMeta, tb_uid_t uid); +int metaCommit(SMeta *pMeta); +int32_t metaCreateTSma(SMeta *pMeta, SSmaCfg *pCfg); +int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid); +STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid); +STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid); +SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline); +STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver); +STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid); +STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid); +SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup); +int metaGetTbNum(SMeta *pMeta); +SMTbCursor *metaOpenTbCursor(SMeta *pMeta); +void metaCloseTbCursor(SMTbCursor *pTbCur); +char *metaTbCursorNext(SMTbCursor *pTbCur); +SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid); +void metaCloseCtbCurosr(SMCtbCursor *pCtbCur); +tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur); + +SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid); +void metaCloseSmaCurosr(SMSmaCursor *pSmaCur); +const char *metaSmaCursorNext(SMSmaCursor *pSmaCur); + +// Options +void metaOptionsInit(SMetaCfg *pMetaCfg); +void metaOptionsClear(SMetaCfg *pMetaCfg); + +// query condition to build multi-table data block iterator +// STsdb +STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta, STfs *pTfs); +void tsdbClose(STsdb *); +void tsdbRemove(const char *path); +int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp); +int tsdbPrepareCommit(STsdb *pTsdb); +int tsdbCommit(STsdb *pTsdb); + + +int32_t tsdbInitSma(STsdb *pTsdb); +int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg); +int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg); +/** + * @brief When submit msg received, update the relative expired window synchronously. + * + * @param pTsdb + * @param msg + * @return int32_t + */ +int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, SSubmitReq *pMsg); + +/** + * @brief Insert tSma(Time-range-wise SMA) data from stream computing engine + * + * @param pTsdb + * @param indexUid + * @param msg + * @return int32_t + */ +int32_t tsdbInsertTSmaData(STsdb *pTsdb, int64_t indexUid, const char *msg); + +/** + * @brief Drop tSma data and local cache. + * + * @param pTsdb + * @param indexUid + * @return int32_t + */ +int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid); + +/** + * @brief Insert RSma(Rollup SMA) data. + * + * @param pTsdb + * @param msg + * @return int32_t + */ +int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg); + +// TODO: This is the basic params, and should wrap the params to a queryHandle. +/** + * @brief Get tSma(Time-range-wise SMA) data. + * + * @param pTsdb + * @param pData + * @param indexUid + * @param querySKey + * @param nMaxResult + * @return int32_t + */ +int32_t tsdbGetTSmaData(STsdb *pTsdb, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult); + +// STsdbCfg +int tsdbOptionsInit(STsdbCfg *); +void tsdbOptionsClear(STsdbCfg *); + +typedef void* tsdbReaderT; + +/** + * Get the data block iterator, starting from position according to the query condition + * + * @param tsdb tsdb handle + * @param pCond query condition, including time window, result set order, and basic required columns for each block + * @param tableInfoGroup table object list in the form of set, grouped into different sets according to the + * group by condition + * @param qinfo query info handle from query processor + * @return + */ +tsdbReaderT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, uint64_t taskId); + +/** + * Get the last row of the given query time window for all the tables in STableGroupInfo object. + * Note that only one data block with only row will be returned while invoking retrieve data block function for + * all tables in this group. + * + * @param tsdb tsdb handle + * @param pCond query condition, including time window, result set order, and basic required columns for each block + * @param tableInfo table list. + * @return + */ +//tsdbReaderT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, uint64_t qId, +// SMemRef *pRef); + + +tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, void* pMemRef); + +int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* pReader, STableBlockDistInfo* pTableBlockInfo); + +bool isTsdbCacheLastRow(tsdbReaderT* pReader); + +/** + * + * @param tsdb + * @param uid + * @param skey + * @param pTagCond + * @param len + * @param tagNameRelType + * @param tbnameCond + * @param pGroupInfo + * @param pColIndex + * @param numOfCols + * @param reqId + * @return + */ +int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, + int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo, + SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId, uint64_t taskId); +/** + * get num of rows in mem table + * + * @param pHandle + * @return row size + */ + +int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT* pHandle); + +/** + * move to next block if exists + * + * @param pTsdbReadHandle + * @return + */ +bool tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle); + +/** + * Get current data block information + * + * @param pTsdbReadHandle + * @param pBlockInfo + * @return + */ +void tsdbRetrieveDataBlockInfo(tsdbReaderT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo); + +/** + * + * Get the pre-calculated information w.r.t. current data block. + * + * In case of data block in cache, the pBlockStatis will always be NULL. + * If a block is not completed loaded from disk, the pBlockStatis will be NULL. + + * @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0 + * @return + */ +int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT *pTsdbReadHandle, SDataStatis **pBlockStatis); + +/** + * + * The query condition with primary timestamp is passed to iterator during its constructor function, + * the returned data block must be satisfied with the time window condition in any cases, + * which means the SData data block is not actually the completed disk data blocks. + * + * @param pTsdbReadHandle query handle + * @param pColumnIdList required data columns id list + * @return + */ +SArray *tsdbRetrieveDataBlock(tsdbReaderT *pTsdbReadHandle, SArray *pColumnIdList); + +/** + * destroy the created table group list, which is generated by tag query + * @param pGroupList + */ +void tsdbDestroyTableGroup(STableGroupInfo *pGroupList); + +/** + * create the table group result including only one table, used to handle the normal table query + * + * @param tsdb tsdbHandle + * @param uid table uid + * @param pGroupInfo the generated result + * @return + */ +int32_t tsdbGetOneTableGroup(void *pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo); + +/** + * + * @param tsdb + * @param pTableIdList + * @param pGroupInfo + * @return + */ +int32_t tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo); + +/** + * clean up the query handle + * @param queryHandle + */ +void tsdbCleanupReadHandle(tsdbReaderT queryHandle); + +int32_t tdScanAndConvertSubmitMsg(SSubmitReq *pMsg); + + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/metaDef.h b/source/dnode/vnode/src/inc/meta.h similarity index 94% rename from source/dnode/vnode/src/inc/metaDef.h rename to source/dnode/vnode/src/inc/meta.h index bc1017f0c793ced80165c8adb964a153e583acea..b04364daf861c32b33a5e4f7a13f56f9ccf25b29 100644 --- a/source/dnode/vnode/src/inc/metaDef.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -13,12 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_META_DEF_H_ -#define _TD_META_DEF_H_ - -#include "tmallocator.h" - -#include "meta.h" +#ifndef _TD_VNODE_META_H_ +#define _TD_VNODE_META_H_ #ifdef __cplusplus extern "C" { @@ -78,4 +74,4 @@ struct SMeta { } #endif -#endif /*_TD_META_DEF_H_*/ +#endif /*_TD_VNODE_META_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tq.h similarity index 70% rename from source/dnode/vnode/src/inc/tqInt.h rename to source/dnode/vnode/src/inc/tq.h index 6c4825e0f25bcc1e50cba630e9d35c24d537b0da..bee443f48735561ca4852e23af7f734aac414b07 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -13,18 +13,14 @@ * along with this program. If not, see . */ -#ifndef _TD_TQ_INT_H_ -#define _TD_TQ_INT_H_ - -#include "meta.h" -#include "tlog.h" -#include "tqPush.h" -#include "vnd.h" +#ifndef _TD_VNODE_TQ_H_ +#define _TD_VNODE_TQ_H_ #ifdef __cplusplus extern "C" { #endif +// tqInt.h #define tqFatal(...) \ { \ if (tqDebugFlag & DEBUG_FATAL) { \ @@ -106,6 +102,9 @@ static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST; typedef enum { TQ_ITEM_READY, TQ_ITEM_PROCESS, TQ_ITEM_EMPTY } STqItemStatus; +typedef struct STqOffsetCfg STqOffsetCfg; +typedef struct STqOffsetStore STqOffsetStore; + typedef struct { int16_t ver; int16_t action; @@ -211,13 +210,93 @@ typedef struct { SArray* topics; // SArray } STqConsumer; +enum { + TQ_PUSHER_TYPE__CLIENT = 1, + TQ_PUSHER_TYPE__STREAM, +}; + +typedef struct { + int8_t type; + int8_t reserved[3]; + int32_t ttl; + int64_t consumerId; + SRpcMsg* pMsg; + // SMqPollRsp* rsp; +} STqClientPusher; + +typedef struct { + int8_t type; + int8_t nodeType; + int8_t reserved[6]; + int64_t streamId; + qTaskInfo_t task; + // TODO sync function +} STqStreamPusher; + +typedef struct { + int8_t type; // mq or stream +} STqPusher; + +typedef struct { + SHashObj* pHash; // +} STqPushMgr; + +typedef struct { + int8_t inited; + tmr_h timer; +} STqPushMgmt; + +static STqPushMgmt tqPushMgmt; + + int32_t tqSerializeConsumer(const STqConsumer*, STqSerializedHead**); int32_t tqDeserializeConsumer(STQ*, const STqSerializedHead*, STqConsumer**); static int FORCE_INLINE tqQueryExecuting(int32_t status) { return status; } +// tqMetaStore.h +STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize pSerializer, FTqDeserialize pDeserializer, + FTqDelete pDeleter, int32_t tqConfigFlag); +int32_t tqStoreClose(STqMetaStore*); +// int32_t tqStoreDelete(TqMetaStore*); +// int32_t tqStoreCommitAll(TqMetaStore*); +int32_t tqStorePersist(STqMetaStore*); +// clean deleted idx and data from persistent file +int32_t tqStoreCompact(STqMetaStore*); + +void* tqHandleGet(STqMetaStore*, int64_t key); +// make it unpersist +void* tqHandleTouchGet(STqMetaStore*, int64_t key); +int32_t tqHandleMovePut(STqMetaStore*, int64_t key, void* value); +int32_t tqHandleCopyPut(STqMetaStore*, int64_t key, void* value, size_t vsize); +// delete committed kv pair +// notice that a delete action still needs to be committed +int32_t tqHandleDel(STqMetaStore*, int64_t key); +int32_t tqHandlePurge(STqMetaStore*, int64_t key); +int32_t tqHandleCommit(STqMetaStore*, int64_t key); +int32_t tqHandleAbort(STqMetaStore*, int64_t key); + +// tqOffset +STqOffsetStore* STqOffsetOpen(STqOffsetCfg*); +void STqOffsetClose(STqOffsetStore*); + +int64_t tqOffsetFetch(STqOffsetStore* pStore, const char* subscribeKey); +int32_t tqOffsetCommit(STqOffsetStore* pStore, const char* subscribeKey, int64_t offset); +int32_t tqOffsetPersist(STqOffsetStore* pStore, const char* subscribeKey); +int32_t tqOffsetPersistAll(STqOffsetStore* pStore); + +// tqPush +int32_t tqPushMgrInit(); +void tqPushMgrCleanUp(); + +STqPushMgr* tqPushMgrOpen(); +void tqPushMgrClose(STqPushMgr* pushMgr); + +STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl); +STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet); + #ifdef __cplusplus } #endif -#endif /*_TD_TQ_INT_H_*/ +#endif /*_TD_VNODE_TQ_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/tqCommit.h b/source/dnode/vnode/src/inc/tqCommit.h deleted file mode 100644 index f2f48bbc8a69a022d0fc6b8a88c5a9a55d0b4ad6..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/src/inc/tqCommit.h +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ diff --git a/source/dnode/vnode/src/inc/tqMetaStore.h b/source/dnode/vnode/src/inc/tqMetaStore.h deleted file mode 100644 index eb203b711778326d84e4daac40113a84e67690a7..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/src/inc/tqMetaStore.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TQ_META_STORE_H_ -#define _TQ_META_STORE_H_ - -#include "os.h" -#include "tqInt.h" - -#ifdef __cplusplus -extern "C" { -#endif - -STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize pSerializer, FTqDeserialize pDeserializer, - FTqDelete pDeleter, int32_t tqConfigFlag); -int32_t tqStoreClose(STqMetaStore*); -// int32_t tqStoreDelete(TqMetaStore*); -// int32_t tqStoreCommitAll(TqMetaStore*); -int32_t tqStorePersist(STqMetaStore*); -// clean deleted idx and data from persistent file -int32_t tqStoreCompact(STqMetaStore*); - -void* tqHandleGet(STqMetaStore*, int64_t key); -// make it unpersist -void* tqHandleTouchGet(STqMetaStore*, int64_t key); -int32_t tqHandleMovePut(STqMetaStore*, int64_t key, void* value); -int32_t tqHandleCopyPut(STqMetaStore*, int64_t key, void* value, size_t vsize); -// delete committed kv pair -// notice that a delete action still needs to be committed -int32_t tqHandleDel(STqMetaStore*, int64_t key); -int32_t tqHandlePurge(STqMetaStore*, int64_t key); -int32_t tqHandleCommit(STqMetaStore*, int64_t key); -int32_t tqHandleAbort(STqMetaStore*, int64_t key); - -#ifdef __cplusplus -} -#endif - -#endif /* ifndef _TQ_META_STORE_H_ */ diff --git a/source/dnode/vnode/src/inc/tqOffset.h b/source/dnode/vnode/src/inc/tqOffset.h deleted file mode 100644 index b58de26f686fe03981370ab15dd4dd7b19c9e2ea..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/src/inc/tqOffset.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TQ_OFFSET_H_ -#define _TD_TQ_OFFSET_H_ - -#include "tqInt.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct STqOffsetCfg STqOffsetCfg; -typedef struct STqOffsetStore STqOffsetStore; - -STqOffsetStore* STqOffsetOpen(STqOffsetCfg*); -void STqOffsetClose(STqOffsetStore*); - -int64_t tqOffsetFetch(STqOffsetStore* pStore, const char* subscribeKey); -int32_t tqOffsetCommit(STqOffsetStore* pStore, const char* subscribeKey, int64_t offset); -int32_t tqOffsetPersist(STqOffsetStore* pStore, const char* subscribeKey); -int32_t tqOffsetPersistAll(STqOffsetStore* pStore); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TQ_OFFSET_H_*/ diff --git a/source/dnode/vnode/src/inc/tqPush.h b/source/dnode/vnode/src/inc/tqPush.h deleted file mode 100644 index a6121c5dc1b7e5b29626a2b6cb59bd9bd63b40b9..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/src/inc/tqPush.h +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TQ_PUSH_H_ -#define _TQ_PUSH_H_ - -#include "executor.h" -#include "thash.h" -#include "trpc.h" -#include "ttimer.h" -#include "vnode.h" - -#ifdef __cplusplus -extern "C" { -#endif - -enum { - TQ_PUSHER_TYPE__CLIENT = 1, - TQ_PUSHER_TYPE__STREAM, -}; - -typedef struct { - int8_t type; - int8_t reserved[3]; - int32_t ttl; - int64_t consumerId; - SRpcMsg* pMsg; - // SMqPollRsp* rsp; -} STqClientPusher; - -typedef struct { - int8_t type; - int8_t nodeType; - int8_t reserved[6]; - int64_t streamId; - qTaskInfo_t task; - // TODO sync function -} STqStreamPusher; - -typedef struct { - int8_t type; // mq or stream -} STqPusher; - -typedef struct { - SHashObj* pHash; // -} STqPushMgr; - -typedef struct { - int8_t inited; - tmr_h timer; -} STqPushMgmt; - -static STqPushMgmt tqPushMgmt; - -int32_t tqPushMgrInit(); -void tqPushMgrCleanUp(); - -STqPushMgr* tqPushMgrOpen(); -void tqPushMgrClose(STqPushMgr* pushMgr); - -STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl); -STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet); - -#ifdef __cplusplus -} -#endif - -#endif /*_TQ_PUSH_H_*/ diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h new file mode 100644 index 0000000000000000000000000000000000000000..97a0832b9c893b496992ecfa60139be92f40129a --- /dev/null +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -0,0 +1,936 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_VNODE_TSDB_H_ +#define _TD_VNODE_TSDB_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SSmaStat SSmaStat; +typedef struct SSmaEnv SSmaEnv; +typedef struct SSmaEnvs SSmaEnvs; + +typedef enum { + TSDB_FILE_HEAD = 0, // .head + TSDB_FILE_DATA, // .data + TSDB_FILE_LAST, // .last + TSDB_FILE_SMAD, // .smad(Block-wise SMA) + TSDB_FILE_SMAL, // .smal(Block-wise SMA) + TSDB_FILE_MAX, // + TSDB_FILE_META, // meta + TSDB_FILE_TSMA, // v2t100.${sma_index_name}, Time-range-wise SMA + TSDB_FILE_RSMA, // v2r100.${sma_index_name}, Time-range-wise Rollup SMA +} E_TSDB_FILE_T; + +typedef struct { + uint32_t magic; + uint32_t fver; + uint32_t len; + uint32_t totalBlocks; + uint32_t totalSubBlocks; + uint32_t offset; + uint64_t size; + uint64_t tombSize; +} SDFInfo; + +typedef struct { + SDFInfo info; + STfsFile f; + TdFilePtr pFile; + uint8_t state; +} SDFile; + +struct SSmaEnvs { + int16_t nTSma; + int16_t nRSma; + SSmaEnv *pTSmaEnv; + SSmaEnv *pRSmaEnv; +}; + +typedef struct { + int fid; + int8_t state; // -128~127 + uint8_t ver; // 0~255, DFileSet version + uint16_t reserve; + SDFile files[TSDB_FILE_MAX]; +} SDFileSet; + +typedef struct { + int minFid; + int midFid; + int maxFid; + TSKEY minKey; +} SRtn; + +typedef struct STbData { + tb_uid_t uid; + TSKEY keyMin; + TSKEY keyMax; + int64_t nrows; + SSkipList *pData; +} STbData; + +typedef struct STsdbMemTable { + T_REF_DECLARE() + SRWLatch latch; + TSKEY keyMin; + TSKEY keyMax; + uint64_t nRow; + SMemAllocator *pMA; + // Container + SSkipList *pSlIdx; // SSkiplist + SHashObj * pHashIdx; +} STsdbMemTable; + +typedef struct { + uint32_t version; // Commit version from 0 to increase + int64_t totalPoints; // total points + int64_t totalStorage; // Uncompressed total storage +} STsdbFSMeta; + +// ================== +typedef struct { + STsdbFSMeta meta; // FS meta + SArray * df; // data file array + SArray * sf; // sma data file array v2f1900.index_name_1 +} SFSStatus; + +typedef struct { + TdThreadRwlock lock; + + SFSStatus *cstatus; // current status + SHashObj * metaCache; // meta cache + SHashObj * metaCacheComp; // meta cache for compact + bool intxn; + SFSStatus *nstatus; // new status +} STsdbFS; + +struct STsdb { + int32_t vgId; + bool repoLocked; + TdThreadMutex mutex; + char * path; + STsdbCfg config; + STsdbMemTable * mem; + STsdbMemTable * imem; + SRtn rtn; + SMemAllocatorFactory *pmaf; + STsdbFS * fs; + SMeta * pMeta; + STfs * pTfs; + SSmaEnvs smaEnvs; +}; + +#define REPO_ID(r) ((r)->vgId) +#define REPO_CFG(r) (&(r)->config) +#define REPO_FS(r) ((r)->fs) +#define REPO_META(r) ((r)->pMeta) +#define REPO_TFS(r) ((r)->pTfs) +#define IS_REPO_LOCKED(r) ((r)->repoLocked) +#define REPO_TSMA_NUM(r) ((r)->smaEnvs.nTSma) +#define REPO_RSMA_NUM(r) ((r)->smaEnvs.nRSma) +#define REPO_TSMA_ENV(r) ((r)->smaEnvs.pTSmaEnv) +#define REPO_RSMA_ENV(r) ((r)->smaEnvs.pRSmaEnv) + +int tsdbLockRepo(STsdb *pTsdb); +int tsdbUnlockRepo(STsdb *pTsdb); + +static FORCE_INLINE STSchema *tsdbGetTableSchemaImpl(STable *pTable, bool lock, bool copy, int32_t version) { + return pTable->pSchema; +} + +// tsdbLog +extern int32_t tsdbDebugFlag; + +#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TSDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0) +#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TSDB ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0) +#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TSDB WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0) +#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TSDB ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0) +#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSDB ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0) +#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) + +// tsdbMemTable.h +typedef struct { + int rowsInserted; + int rowsUpdated; + int rowsDeleteSucceed; + int rowsDeleteFailed; + int nOperations; + TSKEY keyFirst; + TSKEY keyLast; +} SMergeInfo; + +static void * taosTMalloc(size_t size); +static void * taosTCalloc(size_t nmemb, size_t size); +static void * taosTRealloc(void *ptr, size_t size); +static void * taosTZfree(void *ptr); +static size_t taosTSizeof(void *ptr); +static void taosTMemset(void *ptr, int c); + +STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb); +void tsdbFreeMemTable(STsdb *pTsdb, STsdbMemTable *pMemTable); +int tsdbMemTableInsert(STsdb *pTsdb, STsdbMemTable *pMemTable, SSubmitReq *pMsg, SSubmitRsp *pRsp); +int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, + TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo); + +static FORCE_INLINE STSRow *tsdbNextIterRow(SSkipListIterator *pIter) { + if (pIter == NULL) return NULL; + + SSkipListNode *node = tSkipListIterGet(pIter); + if (node == NULL) return NULL; + + return (STSRow *)SL_GET_NODE_DATA(node); +} + +static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator *pIter) { + STSRow *row = tsdbNextIterRow(pIter); + if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL; + + return TD_ROW_KEY(row); +} + +// tsdbOptions +extern const STsdbCfg defautlTsdbOptions; + +int tsdbValidateOptions(const STsdbCfg *); +void tsdbOptionsCopy(STsdbCfg *pDest, const STsdbCfg *pSrc); + +// tsdbReadImpl +typedef struct SReadH SReadH; + +typedef struct { + uint32_t len; + uint32_t offset; + uint32_t hasLast : 2; + uint32_t numOfBlocks : 30; + uint64_t uid; + TSKEY maxKey; +} SBlockIdx; + +#ifdef TD_REFACTOR_3 +typedef struct { + int64_t last : 1; + int64_t offset : 63; + int32_t algorithm : 8; + int32_t numOfRows : 24; + int32_t len; + int32_t keyLen; // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols + int16_t numOfSubBlocks; + int16_t numOfCols; // not including timestamp column + TSKEY keyFirst; + TSKEY keyLast; +} SBlock; + +#else + +typedef enum { + TSDB_SBLK_VER_0 = 0, + TSDB_SBLK_VER_MAX, +} ESBlockVer; + +#define SBlockVerLatest TSDB_SBLK_VER_0 + +typedef struct { + uint8_t last : 1; + uint8_t blkVer : 7; + uint8_t numOfSubBlocks; + col_id_t numOfCols; // not including timestamp column + uint32_t len; // data block length + uint32_t keyLen : 20; // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols + uint32_t algorithm : 4; + uint32_t reserve : 8; + col_id_t numOfBSma; + uint16_t numOfRows; + int64_t offset; + uint64_t aggrStat : 1; + uint64_t aggrOffset : 63; + TSKEY keyFirst; + TSKEY keyLast; +} SBlockV0; + +#define SBlock SBlockV0 // latest SBlock definition + +#endif + +typedef struct { + int32_t delimiter; // For recovery usage + int32_t tid; + uint64_t uid; + SBlock blocks[]; +} SBlockInfo; + +#ifdef TD_REFACTOR_3 +typedef struct { + int16_t colId; + uint16_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM + uint16_t reserve : 15; + int32_t len; + uint32_t type : 8; + uint32_t offset : 24; + int64_t sum; + int64_t max; + int64_t min; + int16_t maxIndex; + int16_t minIndex; + int16_t numOfNull; + uint8_t offsetH; + char padding[1]; +} SBlockCol; +#else +typedef struct { + int16_t colId; + uint16_t type : 6; + uint16_t blen : 10; // bitmap length(TODO: full UT for the bitmap compress of various data input) + uint32_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM + uint32_t len : 31; // data length + bitmap length + uint32_t offset; +} SBlockColV0; + +#define SBlockCol SBlockColV0 // latest SBlockCol definition + +typedef struct { + int16_t colId; + int16_t maxIndex; + int16_t minIndex; + int16_t numOfNull; + int64_t sum; + int64_t max; + int64_t min; +} SAggrBlkColV0; + +#define SAggrBlkCol SAggrBlkColV0 // latest SAggrBlkCol definition + +#endif + +// Code here just for back-ward compatibility +static FORCE_INLINE void tsdbSetBlockColOffset(SBlockCol *pBlockCol, uint32_t offset) { +#ifdef TD_REFACTOR_3 + pBlockCol->offset = offset & ((((uint32_t)1) << 24) - 1); + pBlockCol->offsetH = (uint8_t)(offset >> 24); +#else + pBlockCol->offset = offset; +#endif +} + +static FORCE_INLINE uint32_t tsdbGetBlockColOffset(SBlockCol *pBlockCol) { +#ifdef TD_REFACTOR_3 + uint32_t offset1 = pBlockCol->offset; + uint32_t offset2 = pBlockCol->offsetH; + return (offset1 | (offset2 << 24)); +#else + return pBlockCol->offset; +#endif +} + +typedef struct { + int32_t delimiter; // For recovery usage + int32_t numOfCols; // For recovery usage + uint64_t uid; // For recovery usage + SBlockCol cols[]; +} SBlockData; + +typedef void SAggrBlkData; // SBlockCol cols[]; + +struct SReadH { + STsdb *pRepo; + SDFileSet rSet; // FSET to read + SArray *aBlkIdx; // SBlockIdx array + STable *pTable; // table to read + SBlockIdx *pBlkIdx; // current reading table SBlockIdx + int cidx; + SBlockInfo *pBlkInfo; + SBlockData *pBlkData; // Block info + SAggrBlkData *pAggrBlkData; // Aggregate Block info + SDataCols *pDCols[2]; + void *pBuf; // buffer + void *pCBuf; // compression buffer + void *pExBuf; // extra buffer +}; + +#define TSDB_READ_REPO(rh) ((rh)->pRepo) +#define TSDB_READ_REPO_ID(rh) REPO_ID(TSDB_READ_REPO(rh)) +#define TSDB_READ_FSET(rh) (&((rh)->rSet)) +#define TSDB_READ_TABLE(rh) ((rh)->pTable) +#define TSDB_READ_HEAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_HEAD) +#define TSDB_READ_DATA_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_DATA) +#define TSDB_READ_LAST_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_LAST) +#define TSDB_READ_SMAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_SMAD) +#define TSDB_READ_SMAL_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_SMAL) +#define TSDB_READ_BUF(rh) ((rh)->pBuf) +#define TSDB_READ_COMP_BUF(rh) ((rh)->pCBuf) +#define TSDB_READ_EXBUF(rh) ((rh)->pExBuf) + +#define TSDB_BLOCK_STATIS_SIZE(ncols, blkVer) \ + (sizeof(SBlockData) + sizeof(SBlockColV##blkVer) * (ncols) + sizeof(TSCKSUM)) + +static FORCE_INLINE size_t tsdbBlockStatisSize(int nCols, uint32_t blkVer) { + switch (blkVer) { + case TSDB_SBLK_VER_0: + default: + return TSDB_BLOCK_STATIS_SIZE(nCols, 0); + } +} + +#define TSDB_BLOCK_AGGR_SIZE(ncols, blkVer) (sizeof(SAggrBlkColV##blkVer) * (ncols) + sizeof(TSCKSUM)) + +static FORCE_INLINE size_t tsdbBlockAggrSize(int nCols, uint32_t blkVer) { + switch (blkVer) { + case TSDB_SBLK_VER_0: + default: + return TSDB_BLOCK_AGGR_SIZE(nCols, 0); + } +} + +int tsdbInitReadH(SReadH *pReadh, STsdb *pRepo); +void tsdbDestroyReadH(SReadH *pReadh); +int tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet); +void tsdbCloseAndUnsetFSet(SReadH *pReadh); +int tsdbLoadBlockIdx(SReadH *pReadh); +int tsdbSetReadTable(SReadH *pReadh, STable *pTable); +int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget); +int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlockInfo); +int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, + int numOfColsIds); +int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock); +int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx); +void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx); +void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols, SBlock *pBlock); + +static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) { + void *pBuf = *ppBuf; + size_t tsize = taosTSizeof(pBuf); + + if (tsize < size) { + if (tsize == 0) tsize = 1024; + + while (tsize < size) { + tsize *= 2; + } + + *ppBuf = taosTRealloc(pBuf, tsize); + if (*ppBuf == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + } + + return 0; +} + +// tsdbMemory +static FORCE_INLINE void *taosTMalloc(size_t size) { + if (size <= 0) return NULL; + + void *ret = taosMemoryMalloc(size + sizeof(size_t)); + if (ret == NULL) return NULL; + + *(size_t *)ret = size; + + return (void *)((char *)ret + sizeof(size_t)); +} + +static FORCE_INLINE void *taosTCalloc(size_t nmemb, size_t size) { + size_t tsize = nmemb * size; + void * ret = taosTMalloc(tsize); + if (ret == NULL) return NULL; + + taosTMemset(ret, 0); + return ret; +} + +static FORCE_INLINE size_t taosTSizeof(void *ptr) { return (ptr) ? (*(size_t *)((char *)ptr - sizeof(size_t))) : 0; } + +static FORCE_INLINE void taosTMemset(void *ptr, int c) { memset(ptr, c, taosTSizeof(ptr)); } + +static FORCE_INLINE void * taosTRealloc(void *ptr, size_t size) { + if (ptr == NULL) return taosTMalloc(size); + + if (size <= taosTSizeof(ptr)) return ptr; + + void * tptr = (void *)((char *)ptr - sizeof(size_t)); + size_t tsize = size + sizeof(size_t); + void* tptr1 = taosMemoryRealloc(tptr, tsize); + if (tptr1 == NULL) return NULL; + tptr = tptr1; + + *(size_t *)tptr = size; + + return (void *)((char *)tptr + sizeof(size_t)); +} + +static FORCE_INLINE void* taosTZfree(void* ptr) { + if (ptr) { + taosMemoryFree((void*)((char*)ptr - sizeof(size_t))); + } + return NULL; +} + +// tsdbCommit + +typedef struct { + uint64_t uid; + int64_t offset; + int64_t size; +} SKVRecord; + +void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn); + +static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) { + if (key < 0) { + return (int)((key + 1) / tsTickPerDay[precision] / days - 1); + } else { + return (int)((key / tsTickPerDay[precision] / days)); + } +} + +static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) { + if (fid >= pRtn->maxFid) { + return 0; + } else if (fid >= pRtn->midFid) { + return 1; + } else if (fid >= pRtn->minFid) { + return 2; + } else { + return -1; + } +} + +// tsdbDBDef +// typedef struct SDBFile SDBFile; +// typedef DB_ENV* TDBEnv; + +// struct SDBFile { +// int32_t fid; +// DB* pDB; +// char* path; +// }; + +// int32_t tsdbOpenDBF(TDBEnv pEnv, SDBFile* pDBF); +// void tsdbCloseDBF(SDBFile* pDBF); +// int32_t tsdbOpenBDBEnv(DB_ENV** ppEnv, const char* path); +// void tsdbCloseBDBEnv(DB_ENV* pEnv); +// int32_t tsdbSaveSmaToDB(SDBFile* pDBF, void* key, uint32_t keySize, void* data, uint32_t dataSize); +// void* tsdbGetSmaDataByKey(SDBFile* pDBF, void* key, uint32_t keySize, uint32_t* valueSize); + +// tsdbFile +#define TSDB_FILE_HEAD_SIZE 512 +#define TSDB_FILE_DELIMITER 0xF00AFA0F +#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF +#define TSDB_IVLD_FID INT_MIN +#define TSDB_FILE_STATE_OK 0 +#define TSDB_FILE_STATE_BAD 1 + +#define TSDB_FILE_INFO(tf) (&((tf)->info)) +#define TSDB_FILE_F(tf) (&((tf)->f)) +#define TSDB_FILE_PFILE(tf) ((tf)->pFile) +#define TSDB_FILE_FULL_NAME(tf) (TSDB_FILE_F(tf)->aname) +#define TSDB_FILE_OPENED(tf) (TSDB_FILE_PFILE(tf) != NULL) +#define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf)) +#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_PFILE(f) = NULL) +#define TSDB_FILE_LEVEL(tf) (TSDB_FILE_F(tf)->did.level) +#define TSDB_FILE_ID(tf) (TSDB_FILE_F(tf)->did.id) +#define TSDB_FILE_DID(tf) (TSDB_FILE_F(tf)->did) +#define TSDB_FILE_REL_NAME(tf) (TSDB_FILE_F(tf)->rname) +#define TSDB_FILE_ABS_NAME(tf) (TSDB_FILE_F(tf)->aname) +#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_PFILE(tf)) +#define TSDB_FILE_STATE(tf) ((tf)->state) +#define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s)) +#define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK) +#define TSDB_FILE_IS_BAD(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_BAD) + +typedef int32_t TSDB_FILE_T; +typedef enum { + TSDB_FS_VER_0 = 0, + TSDB_FS_VER_MAX, +} ETsdbFsVer; + +#define TSDB_LATEST_FVER TSDB_FS_VER_0 // latest version for DFile +#define TSDB_LATEST_SFS_VER TSDB_FS_VER_0 // latest version for 'current' file + +static FORCE_INLINE uint32_t tsdbGetDFSVersion(TSDB_FILE_T fType) { // latest version for DFile + switch (fType) { + case TSDB_FILE_HEAD: // .head + case TSDB_FILE_DATA: // .data + case TSDB_FILE_LAST: // .last + case TSDB_FILE_SMAD: // .smad(Block-wise SMA) + case TSDB_FILE_SMAL: // .smal(Block-wise SMA) + default: + return TSDB_LATEST_FVER; + } +} + + +void tsdbInitDFile(STsdb *pRepo, SDFile* pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype); +void tsdbInitDFileEx(SDFile* pDFile, SDFile* pODFile); +int tsdbEncodeSDFile(void** buf, SDFile* pDFile); +void* tsdbDecodeSDFile(STsdb *pRepo, void* buf, SDFile* pDFile); +int tsdbCreateDFile(STsdb *pRepo, SDFile* pDFile, bool updateHeader, TSDB_FILE_T fType); +int tsdbUpdateDFileHeader(SDFile* pDFile); +int tsdbLoadDFileHeader(SDFile* pDFile, SDFInfo* pInfo); +int tsdbParseDFilename(const char* fname, int* vid, int* fid, TSDB_FILE_T* ftype, uint32_t* version); + +static FORCE_INLINE void tsdbSetDFileInfo(SDFile* pDFile, SDFInfo* pInfo) { pDFile->info = *pInfo; } + +static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) { + ASSERT(!TSDB_FILE_OPENED(pDFile)); + + pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), flags); + if (pDFile->pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return 0; +} + +static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) { + if (TSDB_FILE_OPENED(pDFile)) { + taosCloseFile(&pDFile->pFile); + TSDB_FILE_SET_CLOSED(pDFile); + } +} + +static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int whence) { + // ASSERT(TSDB_FILE_OPENED(pDFile)); + + int64_t loffset = taosLSeekFile(TSDB_FILE_PFILE(pDFile), offset, whence); + if (loffset < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return loffset; +} + +static FORCE_INLINE int64_t tsdbWriteDFile(SDFile* pDFile, void* buf, int64_t nbyte) { + ASSERT(TSDB_FILE_OPENED(pDFile)); + + int64_t nwrite = taosWriteFile(pDFile->pFile, buf, nbyte); + if (nwrite < nbyte) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return nwrite; +} + +static FORCE_INLINE void tsdbUpdateDFileMagic(SDFile* pDFile, void* pCksm) { + pDFile->info.magic = taosCalcChecksum(pDFile->info.magic, (uint8_t*)(pCksm), sizeof(TSCKSUM)); +} + +static FORCE_INLINE int tsdbAppendDFile(SDFile* pDFile, void* buf, int64_t nbyte, int64_t* offset) { + ASSERT(TSDB_FILE_OPENED(pDFile)); + + int64_t toffset; + + if ((toffset = tsdbSeekDFile(pDFile, 0, SEEK_END)) < 0) { + return -1; + } + + ASSERT(pDFile->info.size == toffset); + + if (offset) { + *offset = toffset; + } + + if (tsdbWriteDFile(pDFile, buf, nbyte) < 0) { + return -1; + } + + pDFile->info.size += nbyte; + + return (int)nbyte; +} + +static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsRemoveFile(TSDB_FILE_F(pDFile)); } + +static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) { + ASSERT(TSDB_FILE_OPENED(pDFile)); + + int64_t nread = taosReadFile(pDFile->pFile, buf, nbyte); + if (nread < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return nread; +} + +static FORCE_INLINE int tsdbCopyDFile(SDFile* pSrc, SDFile* pDest) { + if (tfsCopyFile(TSDB_FILE_F(pSrc), TSDB_FILE_F(pDest)) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + tsdbSetDFileInfo(pDest, TSDB_FILE_INFO(pSrc)); + return 0; +} + +// =============== SDFileSet + +typedef struct { + int fid; + int8_t state; + uint8_t ver; + uint16_t reserve; +#if 0 + SDFInfo info; +#endif + STfsFile f; + TdFilePtr pFile; + +} SSFile; // files split by days with fid + +#define TSDB_LATEST_FSET_VER 0 + +#define TSDB_FSET_FID(s) ((s)->fid) +#define TSDB_FSET_STATE(s) ((s)->state) +#define TSDB_FSET_VER(s) ((s)->ver) +#define TSDB_DFILE_IN_SET(s, t) ((s)->files + (t)) +#define TSDB_FSET_LEVEL(s) TSDB_FILE_LEVEL(TSDB_DFILE_IN_SET(s, 0)) +#define TSDB_FSET_ID(s) TSDB_FILE_ID(TSDB_DFILE_IN_SET(s, 0)) +#define TSDB_FSET_SET_CLOSED(s) \ + do { \ + for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \ + TSDB_FILE_SET_CLOSED(TSDB_DFILE_IN_SET(s, ftype)); \ + } \ + } while (0); +#define TSDB_FSET_FSYNC(s) \ + do { \ + for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \ + TSDB_FILE_FSYNC(TSDB_DFILE_IN_SET(s, ftype)); \ + } \ + } while (0); + +void tsdbInitDFileSet(STsdb *pRepo, SDFileSet* pSet, SDiskID did, int fid, uint32_t ver); +void tsdbInitDFileSetEx(SDFileSet* pSet, SDFileSet* pOSet); +int tsdbEncodeDFileSet(void** buf, SDFileSet* pSet); +void* tsdbDecodeDFileSet(STsdb *pRepo, void* buf, SDFileSet* pSet); +int tsdbEncodeDFileSetEx(void** buf, SDFileSet* pSet); +void* tsdbDecodeDFileSetEx(void* buf, SDFileSet* pSet); +int tsdbApplyDFileSetChange(SDFileSet* from, SDFileSet* to); +int tsdbCreateDFileSet(STsdb *pRepo, SDFileSet* pSet, bool updateHeader); +int tsdbUpdateDFileSetHeader(SDFileSet* pSet); +int tsdbScanAndTryFixDFileSet(STsdb* pRepo, SDFileSet* pSet); + +static FORCE_INLINE void tsdbCloseDFileSet(SDFileSet* pSet) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + tsdbCloseDFile(TSDB_DFILE_IN_SET(pSet, ftype)); + } +} + +static FORCE_INLINE int tsdbOpenDFileSet(SDFileSet* pSet, int flags) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + if (tsdbOpenDFile(TSDB_DFILE_IN_SET(pSet, ftype), flags) < 0) { + tsdbCloseDFileSet(pSet); + return -1; + } + } + return 0; +} + +static FORCE_INLINE void tsdbRemoveDFileSet(SDFileSet* pSet) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + (void)tsdbRemoveDFile(TSDB_DFILE_IN_SET(pSet, ftype)); + } +} + +static FORCE_INLINE int tsdbCopyDFileSet(SDFileSet* pSrc, SDFileSet* pDest) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + if (tsdbCopyDFile(TSDB_DFILE_IN_SET(pSrc, ftype), TSDB_DFILE_IN_SET(pDest, ftype)) < 0) { + tsdbRemoveDFileSet(pDest); + return -1; + } + } + + return 0; +} + +static FORCE_INLINE void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY* minKey, TSKEY* maxKey) { + *minKey = fid * days * tsTickPerDay[precision]; + *maxKey = *minKey + days * tsTickPerDay[precision] - 1; +} + +static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet* pSet) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + if (TSDB_FILE_IS_BAD(TSDB_DFILE_IN_SET(pSet, ftype))) { + return false; + } + } + + return true; +} + +// tsdbFS +// ================== TSDB global config +extern bool tsdbForceKeepFile; + +// ================== CURRENT file header info +typedef struct { + uint32_t version; // Current file system version (relating to code) + uint32_t len; // Encode content length (including checksum) +} SFSHeader; + +// ================== TSDB File System Meta + +/** + * @brief Directory structure of .tsma data files. + * + * /vnode2/tsdb $ tree tsma/ + * tsma/ + * ├── v2f100.index_name_1 + * ├── v2f101.index_name_1 + * ├── v2f102.index_name_1 + * ├── v2f1900.index_name_3 + * ├── v2f1901.index_name_3 + * ├── v2f1902.index_name_3 + * ├── v2f200.index_name_2 + * ├── v2f201.index_name_2 + * └── v2f202.index_name_2 + * + * 0 directories, 9 files + */ + +#define FS_CURRENT_STATUS(pfs) ((pfs)->cstatus) +#define FS_NEW_STATUS(pfs) ((pfs)->nstatus) +#define FS_IN_TXN(pfs) (pfs)->intxn +#define FS_VERSION(pfs) ((pfs)->cstatus->meta.version) +#define FS_TXN_VERSION(pfs) ((pfs)->nstatus->meta.version) + +typedef struct { + int direction; + uint64_t version; // current FS version + STsdbFS * pfs; + int index; // used to position next fset when version the same + int fid; // used to seek when version is changed + SDFileSet *pSet; +} SFSIter; + +#define TSDB_FS_ITER_FORWARD TSDB_ORDER_ASC +#define TSDB_FS_ITER_BACKWARD TSDB_ORDER_DESC + +STsdbFS *tsdbNewFS(const STsdbCfg *pCfg); +void * tsdbFreeFS(STsdbFS *pfs); +int tsdbOpenFS(STsdb *pRepo); +void tsdbCloseFS(STsdb *pRepo); +void tsdbStartFSTxn(STsdb *pRepo, int64_t pointsAdd, int64_t storageAdd); +int tsdbEndFSTxn(STsdb *pRepo); +int tsdbEndFSTxnWithError(STsdbFS *pfs); +void tsdbUpdateFSTxnMeta(STsdbFS *pfs, STsdbFSMeta *pMeta); +// void tsdbUpdateMFile(STsdbFS *pfs, const SMFile *pMFile); +int tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet); + +void tsdbFSIterInit(SFSIter *pIter, STsdbFS *pfs, int direction); +void tsdbFSIterSeek(SFSIter *pIter, int fid); +SDFileSet *tsdbFSIterNext(SFSIter *pIter); +int tsdbLoadMetaCache(STsdb *pRepo, bool recoverMeta); + +static FORCE_INLINE int tsdbRLockFS(STsdbFS *pFs) { + int code = taosThreadRwlockRdlock(&(pFs->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +static FORCE_INLINE int tsdbWLockFS(STsdbFS *pFs) { + int code = taosThreadRwlockWrlock(&(pFs->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) { + int code = taosThreadRwlockUnlock(&(pFs->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +// tsdbSma +// #define TSDB_SMA_TEST // remove after test finished + + +// struct SSmaEnv { +// TdThreadRwlock lock; +// SDiskID did; +// TDBEnv dbEnv; // TODO: If it's better to put it in smaIndex level? +// char *path; // relative path +// SSmaStat *pStat; +// }; + +// #define SMA_ENV_LOCK(env) ((env)->lock) +// #define SMA_ENV_DID(env) ((env)->did) +// #define SMA_ENV_ENV(env) ((env)->dbEnv) +// #define SMA_ENV_PATH(env) ((env)->path) +// #define SMA_ENV_STAT(env) ((env)->pStat) +// #define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems) + + +// void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv); +// void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv); +// #if 0 +// int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result); +// int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin); +// #endif + +// // internal func +// static FORCE_INLINE int32_t tsdbEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) { +// int32_t len = 0; +// len += taosEncodeFixedI64(pData, tsKey); +// len += taosEncodeFixedI64(pData, groupId); +// return len; +// } + +// static FORCE_INLINE int32_t tsdbRLockSma(SSmaEnv *pEnv) { +// int code = taosThreadRwlockRdlock(&(pEnv->lock)); +// if (code != 0) { +// terrno = TAOS_SYSTEM_ERROR(code); +// return -1; +// } +// return 0; +// } + +// static FORCE_INLINE int32_t tsdbWLockSma(SSmaEnv *pEnv) { +// int code = taosThreadRwlockWrlock(&(pEnv->lock)); +// if (code != 0) { +// terrno = TAOS_SYSTEM_ERROR(code); +// return -1; +// } +// return 0; +// } + +// static FORCE_INLINE int32_t tsdbUnLockSma(SSmaEnv *pEnv) { +// int code = taosThreadRwlockUnlock(&(pEnv->lock)); +// if (code != 0) { +// terrno = TAOS_SYSTEM_ERROR(code); +// return -1; +// } +// return 0; +// } + +#ifdef __cplusplus +} +#endif + + +#endif /*_TD_VNODE_TSDB_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/tsdbCommit.h b/source/dnode/vnode/src/inc/tsdbCommit.h deleted file mode 100644 index 2c6dd75e031b6dfca4b7a803e5be651b35ad8d92..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/src/inc/tsdbCommit.h +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TSDB_COMMIT_H_ -#define _TD_TSDB_COMMIT_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - int minFid; - int midFid; - int maxFid; - TSKEY minKey; -} SRtn; - -typedef struct { - uint64_t uid; - int64_t offset; - int64_t size; -} SKVRecord; - -void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn); - -static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) { - if (key < 0) { - return (int)((key + 1) / tsTickPerDay[precision] / days - 1); - } else { - return (int)((key / tsTickPerDay[precision] / days)); - } -} - -static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) { - if (fid >= pRtn->maxFid) { - return 0; - } else if (fid >= pRtn->midFid) { - return 1; - } else if (fid >= pRtn->minFid) { - return 2; - } else { - return -1; - } -} - -#if 0 -#define TSDB_DEFAULT_BLOCK_ROWS(maxRows) ((maxRows)*4 / 5) - -int tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord); -void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord); -void *tsdbCommitData(STsdbRepo *pRepo); -int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn); -int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray *pSubA, void **ppBuf, SBlockIdx *pIdx); -int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf); -int tsdbWriteBlockImpl(STsdbRepo *pRepo, STable *pTable, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock, - bool isLast, bool isSuper, void **ppBuf, void **ppCBuf); -int tsdbApplyRtn(STsdbRepo *pRepo); - -#endif - -#ifdef __cplusplus -} -#endif - -#endif /* _TD_TSDB_COMMIT_H_ */ diff --git a/source/dnode/vnode/src/inc/tsdbCompact.h b/source/dnode/vnode/src/inc/tsdbCompact.h deleted file mode 100644 index c5df4e27acbb9ffb067e1c3af438a35fdd1ecd35..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/src/inc/tsdbCompact.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#if 0 - -#ifndef _TD_TSDB_COMPACT_H_ -#define _TD_TSDB_COMPACT_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -void *tsdbCompactImpl(STsdbRepo *pRepo); - -#ifdef __cplusplus -} -#endif - -#endif /* _TD_TSDB_COMPACT_H_ */ - -#endif \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/tsdbDBDef.h b/source/dnode/vnode/src/inc/tsdbDBDef.h deleted file mode 100644 index ca9b60049eeda2c6788e2675e9d6f2d69d2cf939..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/src/inc/tsdbDBDef.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TSDB_DB_DEF_H_ -#define _TD_TSDB_DB_DEF_H_ - -#include "db.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct SDBFile SDBFile; -typedef DB_ENV* TDBEnv; - -struct SDBFile { - int32_t fid; - DB* pDB; - char* path; -}; - -int32_t tsdbOpenDBF(TDBEnv pEnv, SDBFile* pDBF); -void tsdbCloseDBF(SDBFile* pDBF); -int32_t tsdbOpenBDBEnv(DB_ENV** ppEnv, const char* path); -void tsdbCloseBDBEnv(DB_ENV* pEnv); -int32_t tsdbSaveSmaToDB(SDBFile* pDBF, void* key, uint32_t keySize, void* data, uint32_t dataSize); -void* tsdbGetSmaDataByKey(SDBFile* pDBF, void* key, uint32_t keySize, uint32_t* valueSize); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TSDB_DB_DEF_H_*/ diff --git a/source/dnode/vnode/src/inc/tsdbDef.h b/source/dnode/vnode/src/inc/tsdbDef.h deleted file mode 100644 index 7e79b4a30ccbfd1603dc0d3f1a0133e1bbbdf02a..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/src/inc/tsdbDef.h +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TSDB_DEF_H_ -#define _TD_TSDB_DEF_H_ - -#include "tsdbDBDef.h" -#include "tmallocator.h" -#include "meta.h" -#include "tcompression.h" -#include "tglobal.h" -#include "thash.h" -#include "tlist.h" -#include "tmsg.h" -#include "tskiplist.h" -#include "ttime.h" - -#include "tsdb.h" -#include "tsdbCommit.h" -#include "tsdbFS.h" -#include "tsdbFile.h" -#include "tsdbLog.h" -#include "tsdbMemTable.h" -#include "tsdbMemory.h" -#include "tsdbOptions.h" -#include "tsdbReadImpl.h" -#include "tsdbSma.h" - - -#ifdef __cplusplus -extern "C" { -#endif - -struct STsdb { - int32_t vgId; - bool repoLocked; - TdThreadMutex mutex; - char * path; - STsdbCfg config; - STsdbMemTable * mem; - STsdbMemTable * imem; - SRtn rtn; - SMemAllocatorFactory *pmaf; - STsdbFS * fs; - SMeta * pMeta; - STfs * pTfs; - SSmaEnvs smaEnvs; -}; - -#define REPO_ID(r) ((r)->vgId) -#define REPO_CFG(r) (&(r)->config) -#define REPO_FS(r) ((r)->fs) -#define REPO_META(r) ((r)->pMeta) -#define REPO_TFS(r) ((r)->pTfs) -#define IS_REPO_LOCKED(r) ((r)->repoLocked) -#define REPO_TSMA_NUM(r) ((r)->smaEnvs.nTSma) -#define REPO_RSMA_NUM(r) ((r)->smaEnvs.nRSma) -#define REPO_TSMA_ENV(r) ((r)->smaEnvs.pTSmaEnv) -#define REPO_RSMA_ENV(r) ((r)->smaEnvs.pRSmaEnv) - -int tsdbLockRepo(STsdb *pTsdb); -int tsdbUnlockRepo(STsdb *pTsdb); - -static FORCE_INLINE STSchema *tsdbGetTableSchemaImpl(STable *pTable, bool lock, bool copy, int32_t version) { - return pTable->pSchema; -} - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TSDB_DEF_H_*/ diff --git a/source/dnode/vnode/src/inc/tsdbFS.h b/source/dnode/vnode/src/inc/tsdbFS.h deleted file mode 100644 index 8156cbae00b19115f9bbad5bce8f2393e1daadcc..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/src/inc/tsdbFS.h +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TSDB_FS_H_ -#define _TD_TSDB_FS_H_ - -#include "tsdbFile.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// ================== TSDB global config -extern bool tsdbForceKeepFile; - -// ================== CURRENT file header info -typedef struct { - uint32_t version; // Current file system version (relating to code) - uint32_t len; // Encode content length (including checksum) -} SFSHeader; - -// ================== TSDB File System Meta -typedef struct { - uint32_t version; // Commit version from 0 to increase - int64_t totalPoints; // total points - int64_t totalStorage; // Uncompressed total storage -} STsdbFSMeta; - -// ================== -typedef struct { - STsdbFSMeta meta; // FS meta - SArray * df; // data file array - SArray * sf; // sma data file array v2f1900.index_name_1 -} SFSStatus; - -/** - * @brief Directory structure of .tsma data files. - * - * /vnode2/tsdb $ tree tsma/ - * tsma/ - * ├── v2f100.index_name_1 - * ├── v2f101.index_name_1 - * ├── v2f102.index_name_1 - * ├── v2f1900.index_name_3 - * ├── v2f1901.index_name_3 - * ├── v2f1902.index_name_3 - * ├── v2f200.index_name_2 - * ├── v2f201.index_name_2 - * └── v2f202.index_name_2 - * - * 0 directories, 9 files - */ - -typedef struct { - TdThreadRwlock lock; - - SFSStatus *cstatus; // current status - SHashObj * metaCache; // meta cache - SHashObj * metaCacheComp; // meta cache for compact - bool intxn; - SFSStatus *nstatus; // new status -} STsdbFS; - -#define FS_CURRENT_STATUS(pfs) ((pfs)->cstatus) -#define FS_NEW_STATUS(pfs) ((pfs)->nstatus) -#define FS_IN_TXN(pfs) (pfs)->intxn -#define FS_VERSION(pfs) ((pfs)->cstatus->meta.version) -#define FS_TXN_VERSION(pfs) ((pfs)->nstatus->meta.version) - -typedef struct { - int direction; - uint64_t version; // current FS version - STsdbFS * pfs; - int index; // used to position next fset when version the same - int fid; // used to seek when version is changed - SDFileSet *pSet; -} SFSIter; - -#define TSDB_FS_ITER_FORWARD TSDB_ORDER_ASC -#define TSDB_FS_ITER_BACKWARD TSDB_ORDER_DESC - -STsdbFS *tsdbNewFS(const STsdbCfg *pCfg); -void * tsdbFreeFS(STsdbFS *pfs); -int tsdbOpenFS(STsdb *pRepo); -void tsdbCloseFS(STsdb *pRepo); -void tsdbStartFSTxn(STsdb *pRepo, int64_t pointsAdd, int64_t storageAdd); -int tsdbEndFSTxn(STsdb *pRepo); -int tsdbEndFSTxnWithError(STsdbFS *pfs); -void tsdbUpdateFSTxnMeta(STsdbFS *pfs, STsdbFSMeta *pMeta); -// void tsdbUpdateMFile(STsdbFS *pfs, const SMFile *pMFile); -int tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet); - -void tsdbFSIterInit(SFSIter *pIter, STsdbFS *pfs, int direction); -void tsdbFSIterSeek(SFSIter *pIter, int fid); -SDFileSet *tsdbFSIterNext(SFSIter *pIter); -int tsdbLoadMetaCache(STsdb *pRepo, bool recoverMeta); - -static FORCE_INLINE int tsdbRLockFS(STsdbFS *pFs) { - int code = taosThreadRwlockRdlock(&(pFs->lock)); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - -static FORCE_INLINE int tsdbWLockFS(STsdbFS *pFs) { - int code = taosThreadRwlockWrlock(&(pFs->lock)); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - -static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) { - int code = taosThreadRwlockUnlock(&(pFs->lock)); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - -#ifdef __cplusplus -} -#endif - -#endif /* _TD_TSDB_FS_H_ */ diff --git a/source/dnode/vnode/src/inc/tsdbFile.h b/source/dnode/vnode/src/inc/tsdbFile.h deleted file mode 100644 index e65ef72623f9b12803765c80d63de07f13bbda07..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/src/inc/tsdbFile.h +++ /dev/null @@ -1,435 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TS_TSDB_FILE_H_ -#define _TS_TSDB_FILE_H_ - -#include "tchecksum.h" -#include "tfs.h" - -#ifdef __cplusplus -extern "C" { -#endif - -#define TSDB_FILE_HEAD_SIZE 512 -#define TSDB_FILE_DELIMITER 0xF00AFA0F -#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF -#define TSDB_IVLD_FID INT_MIN -#define TSDB_FILE_STATE_OK 0 -#define TSDB_FILE_STATE_BAD 1 - -#define TSDB_FILE_INFO(tf) (&((tf)->info)) -#define TSDB_FILE_F(tf) (&((tf)->f)) -#define TSDB_FILE_PFILE(tf) ((tf)->pFile) -#define TSDB_FILE_FULL_NAME(tf) (TSDB_FILE_F(tf)->aname) -#define TSDB_FILE_OPENED(tf) (TSDB_FILE_PFILE(tf) != NULL) -#define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf)) -#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_PFILE(f) = NULL) -#define TSDB_FILE_LEVEL(tf) (TSDB_FILE_F(tf)->did.level) -#define TSDB_FILE_ID(tf) (TSDB_FILE_F(tf)->did.id) -#define TSDB_FILE_DID(tf) (TSDB_FILE_F(tf)->did) -#define TSDB_FILE_REL_NAME(tf) (TSDB_FILE_F(tf)->rname) -#define TSDB_FILE_ABS_NAME(tf) (TSDB_FILE_F(tf)->aname) -#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_PFILE(tf)) -#define TSDB_FILE_STATE(tf) ((tf)->state) -#define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s)) -#define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK) -#define TSDB_FILE_IS_BAD(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_BAD) - -typedef enum { - TSDB_FILE_HEAD = 0, // .head - TSDB_FILE_DATA, // .data - TSDB_FILE_LAST, // .last - TSDB_FILE_SMAD, // .smad(Block-wise SMA) - TSDB_FILE_SMAL, // .smal(Block-wise SMA) - TSDB_FILE_MAX, // - TSDB_FILE_META, // meta - TSDB_FILE_TSMA, // v2t100.${sma_index_name}, Time-range-wise SMA - TSDB_FILE_RSMA, // v2r100.${sma_index_name}, Time-range-wise Rollup SMA -} E_TSDB_FILE_T; - -typedef int32_t TSDB_FILE_T; -typedef enum { - TSDB_FS_VER_0 = 0, - TSDB_FS_VER_MAX, -} ETsdbFsVer; - -#define TSDB_LATEST_FVER TSDB_FS_VER_0 // latest version for DFile -#define TSDB_LATEST_SFS_VER TSDB_FS_VER_0 // latest version for 'current' file - -static FORCE_INLINE uint32_t tsdbGetDFSVersion(TSDB_FILE_T fType) { // latest version for DFile - switch (fType) { - case TSDB_FILE_HEAD: // .head - case TSDB_FILE_DATA: // .data - case TSDB_FILE_LAST: // .last - case TSDB_FILE_SMAD: // .smad(Block-wise SMA) - case TSDB_FILE_SMAL: // .smal(Block-wise SMA) - default: - return TSDB_LATEST_FVER; - } -} - -#if 0 -// =============== SMFile -typedef struct { - int64_t size; - int64_t tombSize; - int64_t nRecords; - int64_t nDels; - uint32_t magic; -} SMFInfo; - -typedef struct { - SMFInfo info; - STfsFile f; - int fd; - uint8_t state; -} SMFile; - -void tsdbInitMFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver); -void tsdbInitMFileEx(SMFile* pMFile, const SMFile* pOMFile); -int tsdbEncodeSMFile(void** buf, SMFile* pMFile); -void* tsdbDecodeSMFile(void* buf, SMFile* pMFile); -int tsdbEncodeSMFileEx(void** buf, SMFile* pMFile); -void* tsdbDecodeSMFileEx(void* buf, SMFile* pMFile); -int tsdbApplyMFileChange(SMFile* from, SMFile* to); -int tsdbCreateMFile(SMFile* pMFile, bool updateHeader); -int tsdbUpdateMFileHeader(SMFile* pMFile); -int tsdbLoadMFileHeader(SMFile* pMFile, SMFInfo* pInfo); -int tsdbScanAndTryFixMFile(STsdb* pRepo); -int tsdbEncodeMFInfo(void** buf, SMFInfo* pInfo); -void* tsdbDecodeMFInfo(void* buf, SMFInfo* pInfo); - -static FORCE_INLINE void tsdbSetMFileInfo(SMFile* pMFile, SMFInfo* pInfo) { pMFile->info = *pInfo; } - -static FORCE_INLINE int tsdbOpenMFile(SMFile* pMFile, int flags) { - ASSERT(TSDB_FILE_CLOSED(pMFile)); - - pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), flags); - if (pMFile->fd < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return 0; -} - -static FORCE_INLINE void tsdbCloseMFile(SMFile* pMFile) { - if (TSDB_FILE_OPENED(pMFile)) { - close(pMFile->fd); - TSDB_FILE_SET_CLOSED(pMFile); - } -} - -static FORCE_INLINE int64_t tsdbSeekMFile(SMFile* pMFile, int64_t offset, int whence) { - ASSERT(TSDB_FILE_OPENED(pMFile)); - - int64_t loffset = taosLSeekFile(TSDB_FILE_FD(pMFile), offset, whence); - if (loffset < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return loffset; -} - -static FORCE_INLINE int64_t tsdbWriteMFile(SMFile* pMFile, void* buf, int64_t nbyte) { - ASSERT(TSDB_FILE_OPENED(pMFile)); - - int64_t nwrite = taosWriteFile(pMFile->fd, buf, nbyte); - if (nwrite < nbyte) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return nwrite; -} - -static FORCE_INLINE void tsdbUpdateMFileMagic(SMFile* pMFile, void* pCksum) { - pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t*)(pCksum), sizeof(TSCKSUM)); -} - -static FORCE_INLINE int tsdbAppendMFile(SMFile* pMFile, void* buf, int64_t nbyte, int64_t* offset) { - ASSERT(TSDB_FILE_OPENED(pMFile)); - - int64_t toffset; - - if ((toffset = tsdbSeekMFile(pMFile, 0, SEEK_END)) < 0) { - return -1; - } - - ASSERT(pMFile->info.size == toffset); - - if (offset) { - *offset = toffset; - } - - if (tsdbWriteMFile(pMFile, buf, nbyte) < 0) { - return -1; - } - - pMFile->info.size += nbyte; - - return (int)nbyte; -} - -static FORCE_INLINE int tsdbRemoveMFile(SMFile* pMFile) { return tfsremove(TSDB_FILE_F(pMFile)); } - -static FORCE_INLINE int64_t tsdbReadMFile(SMFile* pMFile, void* buf, int64_t nbyte) { - ASSERT(TSDB_FILE_OPENED(pMFile)); - - int64_t nread = taosReadFile(pMFile->fd, buf, nbyte); - if (nread < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return nread; -} - -#endif - -// =============== SDFile -typedef struct { - uint32_t magic; - uint32_t fver; - uint32_t len; - uint32_t totalBlocks; - uint32_t totalSubBlocks; - uint32_t offset; - uint64_t size; - uint64_t tombSize; -} SDFInfo; - -typedef struct { - SDFInfo info; - STfsFile f; - TdFilePtr pFile; - uint8_t state; -} SDFile; - -void tsdbInitDFile(STsdb *pRepo, SDFile* pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype); -void tsdbInitDFileEx(SDFile* pDFile, SDFile* pODFile); -int tsdbEncodeSDFile(void** buf, SDFile* pDFile); -void* tsdbDecodeSDFile(STsdb *pRepo, void* buf, SDFile* pDFile); -int tsdbCreateDFile(STsdb *pRepo, SDFile* pDFile, bool updateHeader, TSDB_FILE_T fType); -int tsdbUpdateDFileHeader(SDFile* pDFile); -int tsdbLoadDFileHeader(SDFile* pDFile, SDFInfo* pInfo); -int tsdbParseDFilename(const char* fname, int* vid, int* fid, TSDB_FILE_T* ftype, uint32_t* version); - -static FORCE_INLINE void tsdbSetDFileInfo(SDFile* pDFile, SDFInfo* pInfo) { pDFile->info = *pInfo; } - -static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) { - ASSERT(!TSDB_FILE_OPENED(pDFile)); - - pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), flags); - if (pDFile->pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return 0; -} - -static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) { - if (TSDB_FILE_OPENED(pDFile)) { - taosCloseFile(&pDFile->pFile); - TSDB_FILE_SET_CLOSED(pDFile); - } -} - -static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int whence) { - // ASSERT(TSDB_FILE_OPENED(pDFile)); - - int64_t loffset = taosLSeekFile(TSDB_FILE_PFILE(pDFile), offset, whence); - if (loffset < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return loffset; -} - -static FORCE_INLINE int64_t tsdbWriteDFile(SDFile* pDFile, void* buf, int64_t nbyte) { - ASSERT(TSDB_FILE_OPENED(pDFile)); - - int64_t nwrite = taosWriteFile(pDFile->pFile, buf, nbyte); - if (nwrite < nbyte) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return nwrite; -} - -static FORCE_INLINE void tsdbUpdateDFileMagic(SDFile* pDFile, void* pCksm) { - pDFile->info.magic = taosCalcChecksum(pDFile->info.magic, (uint8_t*)(pCksm), sizeof(TSCKSUM)); -} - -static FORCE_INLINE int tsdbAppendDFile(SDFile* pDFile, void* buf, int64_t nbyte, int64_t* offset) { - ASSERT(TSDB_FILE_OPENED(pDFile)); - - int64_t toffset; - - if ((toffset = tsdbSeekDFile(pDFile, 0, SEEK_END)) < 0) { - return -1; - } - - ASSERT(pDFile->info.size == toffset); - - if (offset) { - *offset = toffset; - } - - if (tsdbWriteDFile(pDFile, buf, nbyte) < 0) { - return -1; - } - - pDFile->info.size += nbyte; - - return (int)nbyte; -} - -static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsRemoveFile(TSDB_FILE_F(pDFile)); } - -static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) { - ASSERT(TSDB_FILE_OPENED(pDFile)); - - int64_t nread = taosReadFile(pDFile->pFile, buf, nbyte); - if (nread < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return nread; -} - -static FORCE_INLINE int tsdbCopyDFile(SDFile* pSrc, SDFile* pDest) { - if (tfsCopyFile(TSDB_FILE_F(pSrc), TSDB_FILE_F(pDest)) < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - tsdbSetDFileInfo(pDest, TSDB_FILE_INFO(pSrc)); - return 0; -} - -// =============== SDFileSet -typedef struct { - int fid; - int8_t state; // -128~127 - uint8_t ver; // 0~255, DFileSet version - uint16_t reserve; - SDFile files[TSDB_FILE_MAX]; -} SDFileSet; - -typedef struct { - int fid; - int8_t state; - uint8_t ver; - uint16_t reserve; -#if 0 - SDFInfo info; -#endif - STfsFile f; - TdFilePtr pFile; - -} SSFile; // files split by days with fid - -#define TSDB_LATEST_FSET_VER 0 - -#define TSDB_FSET_FID(s) ((s)->fid) -#define TSDB_FSET_STATE(s) ((s)->state) -#define TSDB_FSET_VER(s) ((s)->ver) -#define TSDB_DFILE_IN_SET(s, t) ((s)->files + (t)) -#define TSDB_FSET_LEVEL(s) TSDB_FILE_LEVEL(TSDB_DFILE_IN_SET(s, 0)) -#define TSDB_FSET_ID(s) TSDB_FILE_ID(TSDB_DFILE_IN_SET(s, 0)) -#define TSDB_FSET_SET_CLOSED(s) \ - do { \ - for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \ - TSDB_FILE_SET_CLOSED(TSDB_DFILE_IN_SET(s, ftype)); \ - } \ - } while (0); -#define TSDB_FSET_FSYNC(s) \ - do { \ - for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \ - TSDB_FILE_FSYNC(TSDB_DFILE_IN_SET(s, ftype)); \ - } \ - } while (0); - -void tsdbInitDFileSet(STsdb *pRepo, SDFileSet* pSet, SDiskID did, int fid, uint32_t ver); -void tsdbInitDFileSetEx(SDFileSet* pSet, SDFileSet* pOSet); -int tsdbEncodeDFileSet(void** buf, SDFileSet* pSet); -void* tsdbDecodeDFileSet(STsdb *pRepo, void* buf, SDFileSet* pSet); -int tsdbEncodeDFileSetEx(void** buf, SDFileSet* pSet); -void* tsdbDecodeDFileSetEx(void* buf, SDFileSet* pSet); -int tsdbApplyDFileSetChange(SDFileSet* from, SDFileSet* to); -int tsdbCreateDFileSet(STsdb *pRepo, SDFileSet* pSet, bool updateHeader); -int tsdbUpdateDFileSetHeader(SDFileSet* pSet); -int tsdbScanAndTryFixDFileSet(STsdb* pRepo, SDFileSet* pSet); - -static FORCE_INLINE void tsdbCloseDFileSet(SDFileSet* pSet) { - for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - tsdbCloseDFile(TSDB_DFILE_IN_SET(pSet, ftype)); - } -} - -static FORCE_INLINE int tsdbOpenDFileSet(SDFileSet* pSet, int flags) { - for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - if (tsdbOpenDFile(TSDB_DFILE_IN_SET(pSet, ftype), flags) < 0) { - tsdbCloseDFileSet(pSet); - return -1; - } - } - return 0; -} - -static FORCE_INLINE void tsdbRemoveDFileSet(SDFileSet* pSet) { - for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - (void)tsdbRemoveDFile(TSDB_DFILE_IN_SET(pSet, ftype)); - } -} - -static FORCE_INLINE int tsdbCopyDFileSet(SDFileSet* pSrc, SDFileSet* pDest) { - for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - if (tsdbCopyDFile(TSDB_DFILE_IN_SET(pSrc, ftype), TSDB_DFILE_IN_SET(pDest, ftype)) < 0) { - tsdbRemoveDFileSet(pDest); - return -1; - } - } - - return 0; -} - -static FORCE_INLINE void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY* minKey, TSKEY* maxKey) { - *minKey = fid * days * tsTickPerDay[precision]; - *maxKey = *minKey + days * tsTickPerDay[precision] - 1; -} - -static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet* pSet) { - for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - if (TSDB_FILE_IS_BAD(TSDB_DFILE_IN_SET(pSet, ftype))) { - return false; - } - } - - return true; -} - -#ifdef __cplusplus -} -#endif - -#endif /* _TS_TSDB_FILE_H_ */ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/tsdbLog.h b/source/dnode/vnode/src/inc/tsdbLog.h deleted file mode 100644 index 3afe62819884c44d69be6a66e3fd51384c829960..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/src/inc/tsdbLog.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TSDB_LOG_H_ -#define _TD_TSDB_LOG_H_ - -#include "tlog.h" - -#ifdef __cplusplus -extern "C" { -#endif - -extern int32_t tsdbDebugFlag; - -#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TSDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0) -#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TSDB ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0) -#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TSDB WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0) -#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TSDB ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0) -#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSDB ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0) -#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) - -#ifdef __cplusplus -} -#endif - -#endif /* _TD_TSDB_LOG_H_ */ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/tsdbMemTable.h b/source/dnode/vnode/src/inc/tsdbMemTable.h deleted file mode 100644 index 0b9d153239daca56cd1a7d547502f4e4d7c9a6b2..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/src/inc/tsdbMemTable.h +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TSDB_MEM_TABLE_H_ -#define _TD_TSDB_MEM_TABLE_H_ - -#include "tsdb.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - int rowsInserted; - int rowsUpdated; - int rowsDeleteSucceed; - int rowsDeleteFailed; - int nOperations; - TSKEY keyFirst; - TSKEY keyLast; -} SMergeInfo; - -typedef struct STbData { - tb_uid_t uid; - TSKEY keyMin; - TSKEY keyMax; - int64_t nrows; - SSkipList *pData; -} STbData; - -typedef struct STsdbMemTable { - T_REF_DECLARE() - SRWLatch latch; - TSKEY keyMin; - TSKEY keyMax; - uint64_t nRow; - SMemAllocator *pMA; - // Container - SSkipList *pSlIdx; // SSkiplist - SHashObj * pHashIdx; -} STsdbMemTable; - -STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb); -void tsdbFreeMemTable(STsdb *pTsdb, STsdbMemTable *pMemTable); -int tsdbMemTableInsert(STsdb *pTsdb, STsdbMemTable *pMemTable, SSubmitReq *pMsg, SSubmitRsp *pRsp); -int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, - TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo); - -static FORCE_INLINE STSRow *tsdbNextIterRow(SSkipListIterator *pIter) { - if (pIter == NULL) return NULL; - - SSkipListNode *node = tSkipListIterGet(pIter); - if (node == NULL) return NULL; - - return (STSRow *)SL_GET_NODE_DATA(node); -} - -static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator *pIter) { - STSRow *row = tsdbNextIterRow(pIter); - if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL; - - return TD_ROW_KEY(row); -} - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TSDB_MEM_TABLE_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/tsdbMemory.h b/source/dnode/vnode/src/inc/tsdbMemory.h deleted file mode 100644 index 69976fc0783117777b5e89a0c33405d6cfb4f0ae..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/src/inc/tsdbMemory.h +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TSDB_MEMORY_H_ -#define _TD_TSDB_MEMORY_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -static void * taosTMalloc(size_t size); -static void * taosTCalloc(size_t nmemb, size_t size); -static void * taosTRealloc(void *ptr, size_t size); -static void * taosTZfree(void *ptr); -static size_t taosTSizeof(void *ptr); -static void taosTMemset(void *ptr, int c); - -static FORCE_INLINE void *taosTMalloc(size_t size) { - if (size <= 0) return NULL; - - void *ret = taosMemoryMalloc(size + sizeof(size_t)); - if (ret == NULL) return NULL; - - *(size_t *)ret = size; - - return (void *)((char *)ret + sizeof(size_t)); -} - -static FORCE_INLINE void *taosTCalloc(size_t nmemb, size_t size) { - size_t tsize = nmemb * size; - void * ret = taosTMalloc(tsize); - if (ret == NULL) return NULL; - - taosTMemset(ret, 0); - return ret; -} - -static FORCE_INLINE size_t taosTSizeof(void *ptr) { return (ptr) ? (*(size_t *)((char *)ptr - sizeof(size_t))) : 0; } - -static FORCE_INLINE void taosTMemset(void *ptr, int c) { memset(ptr, c, taosTSizeof(ptr)); } - -static FORCE_INLINE void * taosTRealloc(void *ptr, size_t size) { - if (ptr == NULL) return taosTMalloc(size); - - if (size <= taosTSizeof(ptr)) return ptr; - - void * tptr = (void *)((char *)ptr - sizeof(size_t)); - size_t tsize = size + sizeof(size_t); - void* tptr1 = taosMemoryRealloc(tptr, tsize); - if (tptr1 == NULL) return NULL; - tptr = tptr1; - - *(size_t *)tptr = size; - - return (void *)((char *)tptr + sizeof(size_t)); -} - -static FORCE_INLINE void* taosTZfree(void* ptr) { - if (ptr) { - taosMemoryFree((void*)((char*)ptr - sizeof(size_t))); - } - return NULL; -} - -#ifdef __cplusplus -} -#endif - -#endif /* _TD_TSDB_MEMORY_H_ */ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/tsdbOptions.h b/source/dnode/vnode/src/inc/tsdbOptions.h deleted file mode 100644 index 46607ea2fe1680084dec32e5224fbb292f8ff571..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/src/inc/tsdbOptions.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TSDB_OPTIONS_H_ -#define _TD_TSDB_OPTIONS_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -extern const STsdbCfg defautlTsdbOptions; - -int tsdbValidateOptions(const STsdbCfg *); -void tsdbOptionsCopy(STsdbCfg *pDest, const STsdbCfg *pSrc); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TSDB_OPTIONS_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/tsdbReadImpl.h b/source/dnode/vnode/src/inc/tsdbReadImpl.h deleted file mode 100644 index 682306221dda21f4b9e6a4b0be5ed88fda238afc..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/src/inc/tsdbReadImpl.h +++ /dev/null @@ -1,256 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TSDB_READ_IMPL_H_ -#define _TD_TSDB_READ_IMPL_H_ - -#include "os.h" -#include "tcommon.h" -#include "tfs.h" -#include "tsdb.h" -#include "tsdbFile.h" -#include "tsdbMemory.h" -#include "tskiplist.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct SReadH SReadH; - -typedef struct { - uint32_t len; - uint32_t offset; - uint32_t hasLast : 2; - uint32_t numOfBlocks : 30; - uint64_t uid; - TSKEY maxKey; -} SBlockIdx; - -#ifdef TD_REFACTOR_3 -typedef struct { - int64_t last : 1; - int64_t offset : 63; - int32_t algorithm : 8; - int32_t numOfRows : 24; - int32_t len; - int32_t keyLen; // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols - int16_t numOfSubBlocks; - int16_t numOfCols; // not including timestamp column - TSKEY keyFirst; - TSKEY keyLast; -} SBlock; - -#else - -typedef enum { - TSDB_SBLK_VER_0 = 0, - TSDB_SBLK_VER_MAX, -} ESBlockVer; - -#define SBlockVerLatest TSDB_SBLK_VER_0 - -typedef struct { - uint8_t last : 1; - uint8_t blkVer : 7; - uint8_t numOfSubBlocks; - col_id_t numOfCols; // not including timestamp column - uint32_t len; // data block length - uint32_t keyLen : 20; // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols - uint32_t algorithm : 4; - uint32_t reserve : 8; - col_id_t numOfBSma; - uint16_t numOfRows; - int64_t offset; - uint64_t aggrStat : 1; - uint64_t aggrOffset : 63; - TSKEY keyFirst; - TSKEY keyLast; -} SBlockV0; - -#define SBlock SBlockV0 // latest SBlock definition - -#endif - -typedef struct { - int32_t delimiter; // For recovery usage - int32_t tid; - uint64_t uid; - SBlock blocks[]; -} SBlockInfo; - -#ifdef TD_REFACTOR_3 -typedef struct { - int16_t colId; - uint16_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM - uint16_t reserve : 15; - int32_t len; - uint32_t type : 8; - uint32_t offset : 24; - int64_t sum; - int64_t max; - int64_t min; - int16_t maxIndex; - int16_t minIndex; - int16_t numOfNull; - uint8_t offsetH; - char padding[1]; -} SBlockCol; -#else -typedef struct { - int16_t colId; - uint16_t type : 6; - uint16_t blen : 10; // bitmap length(TODO: full UT for the bitmap compress of various data input) - uint32_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM - uint32_t len : 31; // data length + bitmap length - uint32_t offset; -} SBlockColV0; - -#define SBlockCol SBlockColV0 // latest SBlockCol definition - -typedef struct { - int16_t colId; - int16_t maxIndex; - int16_t minIndex; - int16_t numOfNull; - int64_t sum; - int64_t max; - int64_t min; -} SAggrBlkColV0; - -#define SAggrBlkCol SAggrBlkColV0 // latest SAggrBlkCol definition - -#endif - -// Code here just for back-ward compatibility -static FORCE_INLINE void tsdbSetBlockColOffset(SBlockCol *pBlockCol, uint32_t offset) { -#ifdef TD_REFACTOR_3 - pBlockCol->offset = offset & ((((uint32_t)1) << 24) - 1); - pBlockCol->offsetH = (uint8_t)(offset >> 24); -#else - pBlockCol->offset = offset; -#endif -} - -static FORCE_INLINE uint32_t tsdbGetBlockColOffset(SBlockCol *pBlockCol) { -#ifdef TD_REFACTOR_3 - uint32_t offset1 = pBlockCol->offset; - uint32_t offset2 = pBlockCol->offsetH; - return (offset1 | (offset2 << 24)); -#else - return pBlockCol->offset; -#endif -} - -typedef struct { - int32_t delimiter; // For recovery usage - int32_t numOfCols; // For recovery usage - uint64_t uid; // For recovery usage - SBlockCol cols[]; -} SBlockData; - -typedef void SAggrBlkData; // SBlockCol cols[]; - -struct SReadH { - STsdb *pRepo; - SDFileSet rSet; // FSET to read - SArray *aBlkIdx; // SBlockIdx array - STable *pTable; // table to read - SBlockIdx *pBlkIdx; // current reading table SBlockIdx - int cidx; - SBlockInfo *pBlkInfo; - SBlockData *pBlkData; // Block info - SAggrBlkData *pAggrBlkData; // Aggregate Block info - SDataCols *pDCols[2]; - void *pBuf; // buffer - void *pCBuf; // compression buffer - void *pExBuf; // extra buffer -}; - -#define TSDB_READ_REPO(rh) ((rh)->pRepo) -#define TSDB_READ_REPO_ID(rh) REPO_ID(TSDB_READ_REPO(rh)) -#define TSDB_READ_FSET(rh) (&((rh)->rSet)) -#define TSDB_READ_TABLE(rh) ((rh)->pTable) -#define TSDB_READ_HEAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_HEAD) -#define TSDB_READ_DATA_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_DATA) -#define TSDB_READ_LAST_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_LAST) -#define TSDB_READ_SMAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_SMAD) -#define TSDB_READ_SMAL_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_SMAL) -#define TSDB_READ_BUF(rh) ((rh)->pBuf) -#define TSDB_READ_COMP_BUF(rh) ((rh)->pCBuf) -#define TSDB_READ_EXBUF(rh) ((rh)->pExBuf) - -#define TSDB_BLOCK_STATIS_SIZE(ncols, blkVer) \ - (sizeof(SBlockData) + sizeof(SBlockColV##blkVer) * (ncols) + sizeof(TSCKSUM)) - -static FORCE_INLINE size_t tsdbBlockStatisSize(int nCols, uint32_t blkVer) { - switch (blkVer) { - case TSDB_SBLK_VER_0: - default: - return TSDB_BLOCK_STATIS_SIZE(nCols, 0); - } -} - -#define TSDB_BLOCK_AGGR_SIZE(ncols, blkVer) (sizeof(SAggrBlkColV##blkVer) * (ncols) + sizeof(TSCKSUM)) - -static FORCE_INLINE size_t tsdbBlockAggrSize(int nCols, uint32_t blkVer) { - switch (blkVer) { - case TSDB_SBLK_VER_0: - default: - return TSDB_BLOCK_AGGR_SIZE(nCols, 0); - } -} - -int tsdbInitReadH(SReadH *pReadh, STsdb *pRepo); -void tsdbDestroyReadH(SReadH *pReadh); -int tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet); -void tsdbCloseAndUnsetFSet(SReadH *pReadh); -int tsdbLoadBlockIdx(SReadH *pReadh); -int tsdbSetReadTable(SReadH *pReadh, STable *pTable); -int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget); -int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlockInfo); -int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, - int numOfColsIds); -int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock); -int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx); -void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx); -void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols, SBlock *pBlock); - -static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) { - void *pBuf = *ppBuf; - size_t tsize = taosTSizeof(pBuf); - - if (tsize < size) { - if (tsize == 0) tsize = 1024; - - while (tsize < size) { - tsize *= 2; - } - - *ppBuf = taosTRealloc(pBuf, tsize); - if (*ppBuf == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - } - - return 0; -} - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TSDB_READ_IMPL_H_*/ diff --git a/source/dnode/vnode/src/inc/tsdbSma.h b/source/dnode/vnode/src/inc/tsdbSma.h deleted file mode 100644 index ebeb67261d413036e22f1017c76fd9ed6fbe4c32..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/src/inc/tsdbSma.h +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TSDB_SMA_H_ -#define _TD_TSDB_SMA_H_ - -#define TSDB_SMA_TEST // remove after test finished - -typedef struct SSmaStat SSmaStat; -typedef struct SSmaEnv SSmaEnv; -typedef struct SSmaEnvs SSmaEnvs; - -struct SSmaEnv { - TdThreadRwlock lock; - SDiskID did; - TDBEnv dbEnv; // TODO: If it's better to put it in smaIndex level? - char *path; // relative path - SSmaStat *pStat; -}; - -#define SMA_ENV_LOCK(env) ((env)->lock) -#define SMA_ENV_DID(env) ((env)->did) -#define SMA_ENV_ENV(env) ((env)->dbEnv) -#define SMA_ENV_PATH(env) ((env)->path) -#define SMA_ENV_STAT(env) ((env)->pStat) -#define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems) - -struct SSmaEnvs { - int16_t nTSma; - int16_t nRSma; - SSmaEnv *pTSmaEnv; - SSmaEnv *pRSmaEnv; -}; - -void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv); -void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv); -#if 0 -int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result); -int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin); -#endif - -// internal func -static FORCE_INLINE int32_t tsdbEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) { - int32_t len = 0; - len += taosEncodeFixedI64(pData, tsKey); - len += taosEncodeFixedI64(pData, groupId); - return len; -} - -static FORCE_INLINE int32_t tsdbRLockSma(SSmaEnv *pEnv) { - int code = taosThreadRwlockRdlock(&(pEnv->lock)); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - -static FORCE_INLINE int32_t tsdbWLockSma(SSmaEnv *pEnv) { - int code = taosThreadRwlockWrlock(&(pEnv->lock)); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - -static FORCE_INLINE int32_t tsdbUnLockSma(SSmaEnv *pEnv) { - int code = taosThreadRwlockUnlock(&(pEnv->lock)); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - -#endif /* _TD_TSDB_SMA_H_ */ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnodeInt.h similarity index 93% rename from source/dnode/vnode/src/inc/vnd.h rename to source/dnode/vnode/src/inc/vnodeInt.h index 598c8c07bcf20c95de14db1c57011aa9f2f56b3b..6e1f00f93190c2220a0961e3ba491e69764d255e 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -16,17 +16,23 @@ #ifndef _TD_VNODE_DEF_H_ #define _TD_VNODE_DEF_H_ -#include "tmallocator.h" -// #include "sync.h" +#include "executor.h" +#include "tchecksum.h" #include "tcoding.h" +#include "tcompression.h" #include "tdatablock.h" #include "tfs.h" +#include "tglobal.h" #include "tlist.h" #include "tlockfree.h" #include "tmacro.h" +#include "tmallocator.h" +#include "tskiplist.h" +#include "ttime.h" +#include "ttimer.h" #include "vnode.h" -#include "vnodeQuery.h" #include "wal.h" +#include "qworker.h" #ifdef __cplusplus extern "C" { @@ -34,8 +40,9 @@ extern "C" { typedef struct STQ STQ; -typedef struct SVState SVState; -typedef struct SVBufPool SVBufPool; +typedef struct SVState SVState; +typedef struct SVBufPool SVBufPool; +typedef struct SQWorkerMgmt SQHandle; typedef struct SVnodeTask { TD_DLIST_NODE(SVnodeTask); @@ -95,7 +102,9 @@ struct SVnode { STfs* pTfs; }; -int vnodeScheduleTask(SVnodeTask* task); +int vnodeScheduleTask(SVnodeTask* task); +int vnodeQueryOpen(SVnode* pVnode); +void vnodeQueryClose(SVnode* pVnode); #define vFatal(...) \ do { \ @@ -205,6 +214,12 @@ int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t wo // sma void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data); +#include "meta.h" + +#include "tsdb.h" + +#include "tq.h" + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/vnodeQuery.h b/source/dnode/vnode/src/inc/vnodeQuery.h deleted file mode 100644 index 7816b4eb46d6a9c97b17bf63d37102167baaa423..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/src/inc/vnodeQuery.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_VNODE_READ_H_ -#define _TD_VNODE_READ_H_ - -#ifdef __cplusplus -extern "C" { -#endif -#include "qworker.h" -#include "vnode.h" - - -typedef struct SQWorkerMgmt SQHandle; - -int vnodeQueryOpen(SVnode *pVnode); -void vnodeQueryClose(SVnode *pVnode); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_VNODE_READ_H_*/ diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 266305cbab7745d7aecdf79292a50fa4b18ab7fa..e1507a3757f017a67810e762123efbdf160ab513 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -13,8 +13,7 @@ * along with this program. If not, see . */ -#include "meta.h" -#include "metaDef.h" +#include "vnodeInt.h" struct SMetaCache { // TODO diff --git a/source/dnode/vnode/src/meta/metaCfg.c b/source/dnode/vnode/src/meta/metaCfg.c index cbaac1c409f7a4cccb8dde32f3590d1b4b737aa3..a5fcb32698d3fdd9ac6795eaab789b6bfbb0bebb 100644 --- a/source/dnode/vnode/src/meta/metaCfg.c +++ b/source/dnode/vnode/src/meta/metaCfg.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "metaDef.h" +#include "vnodeInt.h" const SMetaCfg defaultMetaOptions = {.lruSize = 0}; diff --git a/source/dnode/vnode/src/meta/metaIdx.c b/source/dnode/vnode/src/meta/metaIdx.c index 818da147381c46f1aab1816407861565f071bd81..9a566f788c8733ae0dd20065012ea0f3a30f1385 100644 --- a/source/dnode/vnode/src/meta/metaIdx.c +++ b/source/dnode/vnode/src/meta/metaIdx.c @@ -16,7 +16,7 @@ #ifdef USE_INVERTED_INDEX #include "index.h" #endif -#include "metaDef.h" +#include "vnodeInt.h" struct SMetaIdx { #ifdef USE_INVERTED_INDEX diff --git a/source/dnode/vnode/src/meta/metaMain.c b/source/dnode/vnode/src/meta/metaMain.c index 690b96bbb058a7a74b122a73d9b604da8c18d8bb..ac47c32cbf02131d353541eedb026a61b39bbdf1 100644 --- a/source/dnode/vnode/src/meta/metaMain.c +++ b/source/dnode/vnode/src/meta/metaMain.c @@ -15,7 +15,7 @@ #include "tcoding.h" -#include "metaDef.h" +#include "vnodeInt.h" static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF); static void metaFree(SMeta *pMeta); diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index 36b3b53ccdfe03e886c8a8dcc55e672226c626b3..c78691e7c25099d3e96bac5f7412aba9ac7da230 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "metaDef.h" +#include "vnodeInt.h" #include "tdbInt.h" typedef struct SPoolMem { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 5977ad832c690e5ceb663476dee67b2f42aea7ab..7f06ba88553b0e211db7f79f0e09d87dc2958277 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "metaDef.h" +#include "vnodeInt.h" int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg) { // Validate the tbOptions diff --git a/source/dnode/vnode/src/meta/metaTbCfg.c b/source/dnode/vnode/src/meta/metaTbCfg.c index d71d319a79d4ddd2dc94fb388bbac7030b9ad955..8ecc808786992bfe436716105fec9640b404e680 100644 --- a/source/dnode/vnode/src/meta/metaTbCfg.c +++ b/source/dnode/vnode/src/meta/metaTbCfg.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "metaDef.h" +#include "vnodeInt.h" #include "tcoding.h" int metaValidateTbCfg(SMeta *pMeta, const STbCfg *pTbOptions) { diff --git a/source/dnode/vnode/src/meta/metaTbUid.c b/source/dnode/vnode/src/meta/metaTbUid.c index b488630024044f9c861f8fd0f8d9c8141b5552f6..27a2ecfb195c300998246f52c145bb08d0bf311e 100644 --- a/source/dnode/vnode/src/meta/metaTbUid.c +++ b/source/dnode/vnode/src/meta/metaTbUid.c @@ -13,8 +13,7 @@ * along with this program. If not, see . */ -#include "meta.h" -#include "metaDef.h" +#include "vnodeInt.h" int metaOpenUidGnrt(SMeta *pMeta) { // Init a generator diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 7af5cbc5e5c3af543f6aa1f0c30b605159b68ec7..f1686953834811019d6f8c41076d6bbb872171fa 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -15,9 +15,8 @@ #include "tcompare.h" #include "tdatablock.h" -#include "tqInt.h" -#include "tqMetaStore.h" #include "tstream.h" +#include "vnodeInt.h" int32_t tqInit() { return tqPushMgrInit(); } @@ -83,9 +82,9 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t versi memcpy(data, msg, msgLen); if (msgType == TDMT_VND_SUBMIT) { - if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg) != 0) { - return -1; - } + // if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg) != 0) { + // return -1; + // } } SRpcMsg req = { @@ -272,7 +271,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { fetchOffset = pReq->currentOffset + 1; } - vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch, pTq->pVnode->vgId, pReq->currentOffset, fetchOffset); + vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch, + pTq->pVnode->vgId, pReq->currentOffset, fetchOffset); SMqPollRsp rsp = { /*.consumerId = consumerId,*/ @@ -296,10 +296,10 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { } STqTopic* pTopic = NULL; - int sz = taosArrayGetSize(pConsumer->topics); + int sz = taosArrayGetSize(pConsumer->topics); for (int32_t i = 0; i < sz; i++) { STqTopic* topic = taosArrayGet(pConsumer->topics, i); - //TODO race condition + // TODO race condition ASSERT(pConsumer->consumerId == consumerId); if (strcmp(topic->topicName, pReq->topic) == 0) { pTopic = topic; @@ -307,7 +307,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { } } if (pTopic == NULL) { - vWarn("tmq poll: consumer %ld (epoch %d) topic %s not found in vg %d", consumerId, pReq->epoch, pReq->topic, pTq->pVnode->vgId); + vWarn("tmq poll: consumer %ld (epoch %d) topic %s not found in vg %d", consumerId, pReq->epoch, pReq->topic, + pTq->pVnode->vgId); pMsg->pCont = NULL; pMsg->contLen = 0; pMsg->code = -1; @@ -315,17 +316,17 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { return 0; } - vDebug("poll topic %s from consumer %ld (epoch %d)", pTopic->topicName, consumerId, pReq->epoch); + vDebug("poll topic %s from consumer %ld (epoch %d) %s", pTopic->topicName, consumerId, pReq->epoch, pTopic->topicName); rsp.reqOffset = pReq->currentOffset; rsp.skipLogNum = 0; while (1) { /*if (fetchOffset > walGetLastVer(pTq->pWal) || walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {*/ - //TODO + // TODO consumerEpoch = atomic_load_32(&pConsumer->epoch); if (consumerEpoch > pReq->epoch) { - //TODO: return + // TODO: return break; } SWalReadHead* pHead; diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c index 505687755dd9ff4499590d8fee9388fb8dd09cb9..beb19f48f1f71a0b1b51cc18783d987aef9eaef3 100644 --- a/source/dnode/vnode/src/tq/tqMetaStore.c +++ b/source/dnode/vnode/src/tq/tqMetaStore.c @@ -12,7 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "tqMetaStore.h" +#include "vnodeInt.h" // TODO:replace by an abstract file layer #include #include diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 20270950cdc6ff13208f7d86bcfee48c28dac55e..3cff87340d8464abed14b6b61b4b8670e7b16b3a 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -14,7 +14,7 @@ */ #define _DEFAULT_SOURCE -#include "tqOffset.h" +#include "vnodeInt.h" enum ETqOffsetPersist { TQ_OFFSET_PERSIST__LAZY = 1, @@ -39,4 +39,3 @@ STqOffsetStore* STqOffsetOpen(STqOffsetCfg* pCfg) { pStore->pHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); return pStore; } - diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 7851c071c340aa2fb0bc65cf9cace22166f5bd0d..4384255f8952370a05763faf6ad046662e8301e8 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "tqPush.h" +#include "vnodeInt.h" int32_t tqPushMgrInit() { // diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index a18374015b5f5f32c2e08cc88038328ea8d62c23..fee8bdf27e3836b587a0a0c59e17ca8f36dbf424 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -167,8 +167,8 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) { break; } - if (colDataAppend(pColData, curRow, sVal.val, false) < 0) { - /*if (colDataAppend(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) {*/ + /*if (colDataAppend(pColData, curRow, sVal.val, false) < 0) {*/ + if (colDataAppend(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) { taosArrayDestroyEx(pArray, (void (*)(void*))tDeleteSSDataBlock); return NULL; } diff --git a/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c b/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c index c8f4cd642ac5bea567bf1a470df0a8df42e91095..0deef2e4c9f6de92b68a334434814fb27d3f1a99 100644 --- a/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c @@ -19,8 +19,6 @@ #include "taoserror.h" #include "tcoding.h" #include "thash.h" -#include "tsdbDBDef.h" -#include "tsdbLog.h" #define IMPL_WITH_LOCK 1 diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 18d3de94ccba3cd9af977faf6bfe12c6662f680f..42d29209f5f07060c1fae50646c676f2b1b50235 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "tsdbDef.h" +#include "vnodeInt.h" #define TSDB_MAX_SUBBLOCKS 8 diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index aa235f88de98e950b9bf8730538d2017325b2c6d..1f42e906161fa1c81666a2c4391ccab906d0d7c6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -14,7 +14,7 @@ */ #include -#include "tsdbDef.h" +#include "vnodeInt.h" #include "os.h" typedef enum { TSDB_TXN_TEMP_FILE = 0, TSDB_TXN_CURR_FILE } TSDB_TXN_FILE_T; @@ -1111,7 +1111,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { char bname[TSDB_FILENAME_LEN]; STfsDir * tdir = NULL; const STfsFile *pf = NULL; - const char * pattern = "^v[0-9]+f[0-9]+\\.(head|data|last)(-ver[0-9]+)?$"; + const char * pattern = "^v[0-9]+f[0-9]+\\.(head|data|last|smad|smal)(-ver[0-9]+)?$"; SArray * fArray = NULL; regex_t regex; STsdbFS * pfs = REPO_FS(pRepo); diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 271616e9c2a031cd46dad5e849771510b320d30c..a2dab19141b8da28d2208db4ada03466a2709304 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "tsdbDef.h" +#include "vnodeInt.h" static const char *TSDB_FNAME_SUFFIX[] = { "head", // TSDB_FILE_HEAD diff --git a/source/dnode/vnode/src/tsdb/tsdbMain.c b/source/dnode/vnode/src/tsdb/tsdbMain.c index 653c30c09173b648e6e0adc7587f75d027d2d8d3..8543cde04689e4df8b0ad3646406ae889f4dc391 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMain.c +++ b/source/dnode/vnode/src/tsdb/tsdbMain.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "tsdbDef.h" +#include "vnodeInt.h" static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta, STfs *pTfs); @@ -87,8 +87,8 @@ static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, static void tsdbFree(STsdb *pTsdb) { if (pTsdb) { - tsdbFreeSmaEnv(REPO_TSMA_ENV(pTsdb)); - tsdbFreeSmaEnv(REPO_RSMA_ENV(pTsdb)); + // tsdbFreeSmaEnv(REPO_TSMA_ENV(pTsdb)); + // tsdbFreeSmaEnv(REPO_RSMA_ENV(pTsdb)); tsdbFreeFS(pTsdb->fs); taosMemoryFreeClear(pTsdb->path); taosMemoryFree(pTsdb); @@ -98,7 +98,7 @@ static void tsdbFree(STsdb *pTsdb) { static int tsdbOpenImpl(STsdb *pTsdb) { tsdbOpenFS(pTsdb); - tsdbInitSma(pTsdb); + // tsdbInitSma(pTsdb); // TODO return 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 34dba5d3baee318deeb05bb39dba311aa7e15eda..c657b2e9475c8d7ec8ebbda760b2ba306f2def97 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "tsdbDef.h" +#include "vnodeInt.h" static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg); static int tsdbMemTableInsertTbData(STsdb *pRepo, SSubmitBlk *pBlock, int32_t *pAffectedRows); diff --git a/source/dnode/vnode/src/tsdb/tsdbOptions.c b/source/dnode/vnode/src/tsdb/tsdbOptions.c index 6bca2e30281ef68879ddaf4061978cdd54c2a06a..da7a1d393fa01ef8be07c5e8b0971dfe4677be27 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOptions.c +++ b/source/dnode/vnode/src/tsdb/tsdbOptions.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "tsdbDef.h" +#include "vnodeInt.h" const STsdbCfg defautlTsdbOptions = {.precision = 0, .lruCacheSize = 0, diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 144aa7fcdc69effb04713924bca1ee7ba074bbe8..de6a05df46038360d016347031c82dbc5e01657a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -13,25 +13,18 @@ * along with this program. If not, see . */ -#include "tsdbDef.h" +#include "vnodeInt.h" #include "tdatablock.h" #include "os.h" #include "talgo.h" #include "tcompare.h" #include "tdataformat.h" #include "texception.h" -#include "tsdb.h" -#include "tsdbFS.h" -#include "tsdbLog.h" -#include "tsdbReadImpl.h" -#include "tskiplist.h" -#include "ttime.h" #include "taosdef.h" #include "tlosertree.h" -#include "tsdbDef.h" +#include "vnodeInt.h" #include "tmsg.h" -#include "tsdbCommit.h" #define EXTRA_BYTES 2 #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index 687e2ca776cf1230a46be1867b3123b5faa7815e..8b718dbb0e201b2e1a203191f71fd6f31ef76910 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "tsdbDef.h" +#include "vnodeInt.h" #define TSDB_KEY_COL_OFFSET 0 diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index aacaf1b3ecd9e532d7b5aee2eadf4613b4dc08f7..4ee85e61f1728e4ded925b192b6f7277d3ead447 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "tsdbDef.h" +#include "vnodeInt.h" static const char *TSDB_SMA_DNAME[] = { "", // TSDB_SMA_TYPE_BLOCK diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 5590f13cc6e9c602a5530910de3c0154d547219f..abfcc675eac4df251ecb0da8ace6cfa627754d9a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "tsdbDef.h" +#include "vnodeInt.h" /** * @brief insert TS data diff --git a/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c b/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c index ab0da1451dec1b818290cce00b21d4df62b1af2d..7b7f6c91570ea4bd82dd3b80265eef39ad5ede06 100644 --- a/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c +++ b/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnd.h" +#include "vnodeInt.h" static SVArenaNode *vArenaNodeNew(uint64_t capacity); static void vArenaNodeFree(SVArenaNode *pNode); diff --git a/source/dnode/vnode/src/vnd/vnodeBufferPool.c b/source/dnode/vnode/src/vnd/vnodeBufferPool.c index 5be88cdc2e0fc2b9871a55be8e735e06ba1ddf2f..8764950f27535d97c0fc10658b252f80454e9cb6 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufferPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufferPool.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnd.h" +#include "vnodeInt.h" /* ------------------------ STRUCTURES ------------------------ */ #define VNODE_BUF_POOL_SHARDS 3 diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 3b7206e64d7e88f6e0aabf97be902456cc6be23e..4a70738e864d7668781e99b1e6c708ee522c4e77 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnd.h" +#include "vnodeInt.h" const SVnodeCfg defaultVnodeOptions = { .wsize = 96 * 1024 * 1024, .ssize = 1 * 1024 * 1024, .lsize = 1024, .walCfg = {.level = TAOS_WAL_WRITE}}; /* TODO */ diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 1717721bb2417da0a61cbf36c660521a9d99d38a..1235637e198c62d73e2de99b7773406afc9d23af 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnd.h" +#include "vnodeInt.h" static int vnodeStartCommit(SVnode *pVnode); static int vnodeEndCommit(SVnode *pVnode); diff --git a/source/dnode/vnode/src/vnd/vnodeInt.c b/source/dnode/vnode/src/vnd/vnodeInt.c index 333da7a2e57864a871c0461c08a119467a7d095e..24294c4b58b58478ce0c001134eb22abca71b00b 100644 --- a/source/dnode/vnode/src/vnd/vnodeInt.c +++ b/source/dnode/vnode/src/vnd/vnodeInt.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "sync.h" -#include "vnd.h" +#include "vnodeInt.h" // #include "vnodeInt.h" int32_t vnodeAlter(SVnode *pVnode, const SVnodeCfg *pCfg) { return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeMain.c b/source/dnode/vnode/src/vnd/vnodeMain.c index 91c6e4d263181ada878ba3d9cdc52771233aeb1d..2fd848a39d07f38b2607ad2f0c40b2b67dde54dc 100644 --- a/source/dnode/vnode/src/vnd/vnodeMain.c +++ b/source/dnode/vnode/src/vnd/vnodeMain.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnd.h" +#include "vnodeInt.h" static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg); static void vnodeFree(SVnode *pVnode); diff --git a/source/dnode/vnode/src/vnd/vnodeMgr.c b/source/dnode/vnode/src/vnd/vnodeMgr.c index 8f7d5713ab69be0baceb80f779a1fbb23246e2bb..40f43bcd12cea110c02b84d295cf2b90df1a1ff8 100644 --- a/source/dnode/vnode/src/vnd/vnodeMgr.c +++ b/source/dnode/vnode/src/vnd/vnodeMgr.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "vnd.h" +#include "vnodeInt.h" #include "tglobal.h" SVnodeMgr vnodeMgr = {.vnodeInitFlag = TD_MOD_UNINITIALIZED}; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 30c2fe84bf39eab9eb3dd0275af8927448e84f7d..f56ded9f15a114e349c4d6dbea39832a3010d2df 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -13,9 +13,8 @@ * along with this program. If not, see . */ -#include "vnodeQuery.h" #include "executor.h" -#include "vnd.h" +#include "vnodeInt.h" static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg); static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg); diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index e841eb676192b8fe1390f93f0574d756a9911f99..bf2260c51cbaff653f12468745efa8a9206e69f4 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -13,13 +13,13 @@ * along with this program. If not, see . */ -#include "vnd.h" +#include "vnodeInt.h" void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) { // TODO - blockDebugShowData(data); - tsdbInsertTSmaData(((SVnode *)pVnode)->pTsdb, smaId, (const char *)data); + // blockDebugShowData(data); + // tsdbInsertTSmaData(((SVnode *)pVnode)->pTsdb, smaId, (const char *)data); } void vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { @@ -232,16 +232,16 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { tdDestroyTSma(&vCreateSmaReq.tSma); // TODO: return directly or go on follow steps? #endif - if (tsdbCreateTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { - // TODO - } - } break; - case TDMT_VND_CANCEL_SMA: { // timeRangeSMA - } break; - case TDMT_VND_DROP_SMA: { // timeRangeSMA - if (tsdbDropTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { - // TODO - } + // if (tsdbCreateTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { + // // TODO + // } + // } break; + // case TDMT_VND_CANCEL_SMA: { // timeRangeSMA + // } break; + // case TDMT_VND_DROP_SMA: { // timeRangeSMA + // if (tsdbDropTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { + // // TODO + // } #if 0 tsdbTSmaSub(pVnode->pTsdb, 1); SVDropTSmaReq vDropSmaReq = {0}; diff --git a/source/dnode/vnode/test/CMakeLists.txt b/source/dnode/vnode/test/CMakeLists.txt index af123a3133693d97de16aae71cef049508e47167..59f459462f97f423c2dbdbb7917dbcab1ad97dc0 100644 --- a/source/dnode/vnode/test/CMakeLists.txt +++ b/source/dnode/vnode/test/CMakeLists.txt @@ -25,15 +25,15 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) # COMMAND tqTest # ) -ADD_EXECUTABLE(tsdbSmaTest tsdbSmaTest.cpp) -TARGET_LINK_LIBRARIES( - tsdbSmaTest - PUBLIC os util common vnode gtest_main -) +# ADD_EXECUTABLE(tsdbSmaTest tsdbSmaTest.cpp) +# TARGET_LINK_LIBRARIES( +# tsdbSmaTest +# PUBLIC os util common vnode gtest_main +# ) -TARGET_INCLUDE_DIRECTORIES( - tsdbSmaTest - PUBLIC "${CMAKE_SOURCE_DIR}/include/common" - PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../src/inc" - PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) \ No newline at end of file +# TARGET_INCLUDE_DIRECTORIES( +# tsdbSmaTest +# PUBLIC "${CMAKE_SOURCE_DIR}/include/common" +# PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../src/inc" +# PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +# ) \ No newline at end of file diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index 37e2f188cdf2b83496b98403af7559ccd633ba6f..6a4adfe4f8e33dfdc0f2c421af28beed783442f1 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -14,13 +14,13 @@ */ #include -#include +#include #include #include #include -#include +#include #include #pragma GCC diagnostic push diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 57edc4000723e90164d67a6a7ae8500fff7472ee..35a4b93a880dcef92d1aac31a3be06ba3b57367e 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -511,6 +511,12 @@ typedef struct SProjectOperatorInfo { SSDataBlock *existDataBlock; SArray *pPseudoColInfo; SLimit limit; + SLimit slimit; + + uint64_t groupId; + int64_t curSOffset; + int64_t curGroupOutput; + int64_t curOffset; int64_t curOutput; } SProjectOperatorInfo; @@ -563,6 +569,29 @@ typedef struct SGroupbyOperatorInfo { SAggSupporter aggSup; } SGroupbyOperatorInfo; +typedef struct SDataGroupInfo { + uint64_t groupId; + int64_t numOfRows; + SArray *pPageList; +} SDataGroupInfo; + +// The sort in partition may be needed later. +typedef struct SPartitionOperatorInfo { + SOptrBasicInfo binfo; + SArray* pGroupCols; + SArray* pGroupColVals; // current group column values, SArray + char* keyBuf; // group by keys for hash + int32_t groupKeyLen; // total group by column width + SHashObj* pGroupSet; // quick locate the window object for each result + + SDiskbasedBuf* pBuf; // query result buffer based on blocked-wised disk file + int32_t rowCapacity; // maximum number of rows for each buffer page + int32_t* columnOffset; // start position for each column data + + void* pGroupIter; // group iterator + int32_t pageIndex; // page index of current group +} SPartitionOperatorInfo; + typedef struct SSessionAggOperatorInfo { SOptrBasicInfo binfo; SAggSupporter aggSup; @@ -650,7 +679,7 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); -SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t num, SSDataBlock* pResBlock, SLimit* pLimit, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t num, SSDataBlock* pResBlock, SLimit* pLimit, SLimit* pSlimit, SExecTaskInfo* pTaskInfo); SOperatorInfo *createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataBlock* pResBlock, const SName* pName, @@ -667,8 +696,8 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExp int32_t fillType, char* fillVal, bool multigroupResult, SExecTaskInfo* pTaskInfo); SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResultBlock, SArray* pSortInfo, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); - +SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SArray* pGroupColList, + SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); #if 0 SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index d7c99d54df6557e4de6aec9f8fe64a45a8c8faa6..5cbda9073316e4fb82ae881e50f22fbf35a6cf46 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include +#include #include "dataSinkMgt.h" #include "texception.h" #include "os.h" diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 9c5f901353bddc98d1be43a450131b2fb8f72c2b..e85cfb76fd981311c2b723a3a25cea4c255f7f18 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -34,7 +34,7 @@ #include "tcompare.h" #include "tcompression.h" #include "thash.h" -#include "tsdb.h" +#include "vnode.h" #include "ttypes.h" #include "vnode.h" @@ -302,6 +302,10 @@ SSDataBlock* createOutputBuf_rv1(SDataBlockDescNode* pNode) { } taosArrayPush(pBlock->pDataBlock, &idata); + + if (IS_VAR_DATA_TYPE(idata.info.type)) { + pBlock->info.hasVarCol = true; + } } return pBlock; @@ -1259,6 +1263,7 @@ static void setPseudoOutputColInfo(SSDataBlock* pResult, SqlFunctionCtx* pCtx, S static void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, int32_t numOfOutput, SArray* pPseudoList) { setPseudoOutputColInfo(pResult, pCtx, pPseudoList); + pResult->info.groupId = pSrcBlock->info.groupId; for (int32_t k = 0; k < numOfOutput; ++k) { if (pExpr[k].pExpr->nodeType == QUERY_NODE_COLUMN) { // it is a project query @@ -5422,7 +5427,6 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator, bool* newgroup) publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { - assert(*newgroup == false); *newgroup = prevVal; setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); break; @@ -5450,6 +5454,38 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator, bool* newgroup) projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfOutput, pProjectInfo->pPseudoColInfo); + if (pProjectInfo->curSOffset > 0) { + if (pProjectInfo->groupId == 0) { // it is the first group + pProjectInfo->groupId = pBlock->info.groupId; + blockDataCleanup(pInfo->pRes); + continue; + } else if (pProjectInfo->groupId != pBlock->info.groupId) { + pProjectInfo->curSOffset -= 1; + + // ignore data block in current group + if (pProjectInfo->curSOffset > 0) { + blockDataCleanup(pInfo->pRes); + continue; + } + } + + pProjectInfo->groupId = pBlock->info.groupId; + } + + if (pProjectInfo->groupId != 0 && pProjectInfo->groupId != pBlock->info.groupId) { + pProjectInfo->curGroupOutput += 1; + if ((pProjectInfo->slimit.limit > 0) && (pProjectInfo->slimit.limit <= pProjectInfo->curGroupOutput)) { + pOperator->status = OP_EXEC_DONE; + return NULL; + } + + // reset the value for a new group data + pProjectInfo->curOffset = 0; + pProjectInfo->curOutput = 0; + } + + pProjectInfo->groupId = pBlock->info.groupId; + // todo extract method if (pProjectInfo->curOffset < pInfo->pRes->info.rows && pProjectInfo->curOffset > 0) { blockDataTrimFirstNRows(pInfo->pRes, pProjectInfo->curOffset); @@ -6317,7 +6353,7 @@ static SArray* setRowTsColumnOutputInfo(SqlFunctionCtx* pCtx, int32_t numOfCols) } SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t num, - SSDataBlock* pResBlock, SLimit* pLimit, SExecTaskInfo* pTaskInfo) { + SSDataBlock* pResBlock, SLimit* pLimit, SLimit* pSlimit, SExecTaskInfo* pTaskInfo) { SProjectOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SProjectOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -6325,7 +6361,10 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p } pInfo->limit = *pLimit; + pInfo->slimit = *pSlimit; pInfo->curOffset = pLimit->offset; + pInfo->curSOffset = pSlimit->offset; + pInfo->binfo.pRes = pResBlock; int32_t numOfCols = num; @@ -7052,11 +7091,13 @@ static SArray* extractScanColumnId(SNodeList* pNodeList); static SArray* extractColumnInfo(SNodeList* pNodeList); static SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols); static SArray* createSortInfo(SNodeList* pNodeList); +static SArray* extractPartitionColInfo(SNodeList* pNodeList); SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId, STableGroupInfo* pTableGroupInfo) { if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) { - if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pPhyNode)) { + int32_t type = nodeType(pPhyNode); + if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == type) { SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; int32_t numOfCols = 0; @@ -7064,11 +7105,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SArray* pColList = extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols); return createTableScanOperatorInfo(pDataReader, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pScanPhyNode->reverse, pColList, pScanPhyNode->node.pConditions, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == nodeType(pPhyNode)) { + } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) { SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pPhyNode; SSDataBlock* pResBlock = createOutputBuf_rv1(pExchange->node.pOutputDataBlockDesc); return createExchangeOperatorInfo(pExchange->pSrcEndPoints, pResBlock, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == nodeType(pPhyNode)) { + } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId); @@ -7081,7 +7122,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pResBlock, pColList, tableIdList, pTaskInfo); taosArrayDestroy(tableIdList); return pOperator; - } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == nodeType(pPhyNode)) { + } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode; SSDataBlock* pResBlock = createOutputBuf_rv1(pSysScanPhyNode->scan.node.pOutputDataBlockDesc); @@ -7097,93 +7138,76 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } } - if (QUERY_NODE_PHYSICAL_PLAN_PROJECT == nodeType(pPhyNode)) { - size_t size = LIST_LENGTH(pPhyNode->pChildren); - assert(size == 1); + int32_t type = nodeType(pPhyNode); + size_t size = LIST_LENGTH(pPhyNode->pChildren); + ASSERT(size == 1); - SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0); - SOperatorInfo* op = createOperatorTree(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableGroupInfo); + SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0); + SOperatorInfo* op = createOperatorTree(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableGroupInfo); + if (QUERY_NODE_PHYSICAL_PLAN_PROJECT == type) { int32_t num = 0; SProjectPhysiNode* pProjPhyNode = (SProjectPhysiNode*) pPhyNode; SExprInfo* pExprInfo = createExprInfo(pProjPhyNode->pProjections, NULL, &num); SSDataBlock* pResBlock = createOutputBuf_rv1(pPhyNode->pOutputDataBlockDesc); SLimit limit = {.limit = pProjPhyNode->limit, .offset = pProjPhyNode->offset}; - - return createProjectOperatorInfo(op, pExprInfo, num, pResBlock, &limit, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_AGG == nodeType(pPhyNode)) { - size_t size = LIST_LENGTH(pPhyNode->pChildren); - assert(size == 1); - - for (int32_t i = 0; i < size; ++i) { - SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i); - SOperatorInfo* op = createOperatorTree(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableGroupInfo); - - int32_t num = 0; - - SAggPhysiNode* pAggNode = (SAggPhysiNode*)pPhyNode; - SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num); - SSDataBlock* pResBlock = createOutputBuf_rv1(pPhyNode->pOutputDataBlockDesc); - - if (pAggNode->pGroupKeys != NULL) { - SArray* pColList = extractColumnInfo(pAggNode->pGroupKeys); - return createGroupOperatorInfo(op, pExprInfo, num, pResBlock, pColList, pAggNode->node.pConditions, pTaskInfo, NULL); - } else { - return createAggregateOperatorInfo(op, pExprInfo, num, pResBlock, pTaskInfo, pTableGroupInfo); - } - } - } else if (QUERY_NODE_PHYSICAL_PLAN_INTERVAL == nodeType(pPhyNode)) { - size_t size = LIST_LENGTH(pPhyNode->pChildren); - assert(size == 1); - - for (int32_t i = 0; i < size; ++i) { - SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i); - SOperatorInfo* op = createOperatorTree(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableGroupInfo); - - SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; - - int32_t num = 0; - SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num); - SSDataBlock* pResBlock = createOutputBuf_rv1(pPhyNode->pOutputDataBlockDesc); - - SInterval interval = { - .interval = pIntervalPhyNode->interval, - .sliding = pIntervalPhyNode->sliding, - .intervalUnit = pIntervalPhyNode->intervalUnit, - .slidingUnit = pIntervalPhyNode->slidingUnit, - .offset = pIntervalPhyNode->offset, - .precision = pIntervalPhyNode->precision - }; - - int32_t primaryTsSlotId = ((SColumnNode*) pIntervalPhyNode->pTspk)->slotId; - return createIntervalOperatorInfo(op, pExprInfo, num, pResBlock, &interval, primaryTsSlotId, pTableGroupInfo, pTaskInfo); + SLimit slimit = {.limit = pProjPhyNode->slimit, .offset = pProjPhyNode->soffset}; + return createProjectOperatorInfo(op, pExprInfo, num, pResBlock, &limit, &slimit, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_AGG == type) { + int32_t num = 0; + + SAggPhysiNode* pAggNode = (SAggPhysiNode*)pPhyNode; + SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num); + SSDataBlock* pResBlock = createOutputBuf_rv1(pPhyNode->pOutputDataBlockDesc); + + if (pAggNode->pGroupKeys != NULL) { + SArray* pColList = extractColumnInfo(pAggNode->pGroupKeys); + return createGroupOperatorInfo(op, pExprInfo, num, pResBlock, pColList, pAggNode->node.pConditions, pTaskInfo, NULL); + } else { + return createAggregateOperatorInfo(op, pExprInfo, num, pResBlock, pTaskInfo, pTableGroupInfo); } - } else if (QUERY_NODE_PHYSICAL_PLAN_SORT == nodeType(pPhyNode)) { - size_t size = LIST_LENGTH(pPhyNode->pChildren); - assert(size == 1); + } else if (QUERY_NODE_PHYSICAL_PLAN_INTERVAL == type) { + SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; - SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0); - SOperatorInfo* op = createOperatorTree(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableGroupInfo); + int32_t num = 0; + SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num); + SSDataBlock* pResBlock = createOutputBuf_rv1(pPhyNode->pOutputDataBlockDesc); + SInterval interval = { + .interval = pIntervalPhyNode->interval, + .sliding = pIntervalPhyNode->sliding, + .intervalUnit = pIntervalPhyNode->intervalUnit, + .slidingUnit = pIntervalPhyNode->slidingUnit, + .offset = pIntervalPhyNode->offset, + .precision = pIntervalPhyNode->precision + }; + + int32_t primaryTsSlotId = ((SColumnNode*) pIntervalPhyNode->pTspk)->slotId; + return createIntervalOperatorInfo(op, pExprInfo, num, pResBlock, &interval, primaryTsSlotId, pTableGroupInfo, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_SORT == type) { SSortPhysiNode* pSortPhyNode = (SSortPhysiNode*)pPhyNode; SSDataBlock* pResBlock = createOutputBuf_rv1(pPhyNode->pOutputDataBlockDesc); - SArray* info = createSortInfo(pSortPhyNode->pSortKeys); - return createSortOperatorInfo(op, pResBlock, info, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW == nodeType(pPhyNode)) { - size_t size = LIST_LENGTH(pPhyNode->pChildren); - assert(size == 1); - - SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0); - SOperatorInfo* op = createOperatorTree(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableGroupInfo); + SArray* info = createSortInfo(pSortPhyNode->pSortKeys); + return createSortOperatorInfo(op, pResBlock, info, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW == type) { SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &num); SSDataBlock* pResBlock = createOutputBuf_rv1(pPhyNode->pOutputDataBlockDesc); return createSessionAggOperatorInfo(op, pExprInfo, num, pResBlock, pSessionNode->gap, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_PARTITION == type) { + SPartitionPhysiNode* pPartNode = (SPartitionPhysiNode*) pPhyNode; + SArray* pColList = extractPartitionColInfo(pPartNode->pPartitionKeys); + SSDataBlock* pResBlock = createOutputBuf_rv1(pPhyNode->pOutputDataBlockDesc); + + int32_t num = 0; + SExprInfo* pExprInfo = createExprInfo(pPartNode->pTargets, NULL, &num); + + return createPartitionOperatorInfo(op, pExprInfo, num, pResBlock, pColList, pTaskInfo, NULL); } else { ASSERT(0); } /*else if (pPhyNode->info.type == OP_MultiTableAggregate) { @@ -7266,11 +7290,38 @@ SArray* extractColumnInfo(SNodeList* pNodeList) { STargetNode* pNode = (STargetNode*)nodesListGetNode(pNodeList, i); SColumnNode* pColNode = (SColumnNode*)pNode->pExpr; + // todo extract method SColumn c = {0}; c.slotId = pColNode->slotId; - c.colId = pColNode->colId; - c.type = pColNode->node.resType.type; - c.bytes = pColNode->node.resType.bytes; + c.colId = pColNode->colId; + c.type = pColNode->node.resType.type; + c.bytes = pColNode->node.resType.bytes; + c.precision = pColNode->node.resType.precision; + c.scale = pColNode->node.resType.scale; + + taosArrayPush(pList, &c); + } + + return pList; +} + +SArray* extractPartitionColInfo(SNodeList* pNodeList) { + size_t numOfCols = LIST_LENGTH(pNodeList); + SArray* pList = taosArrayInit(numOfCols, sizeof(SColumn)); + if (pList == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnNode* pColNode = (SColumnNode*)nodesListGetNode(pNodeList, i); + + // todo extract method + SColumn c = {0}; + c.slotId = pColNode->slotId; + c.colId = pColNode->colId; + c.type = pColNode->node.resType.type; + c.bytes = pColNode->node.resType.bytes; c.precision = pColNode->node.resType.precision; c.scale = pColNode->node.resType.scale; @@ -7296,15 +7347,6 @@ SArray* createSortInfo(SNodeList* pNodeList) { SColumnNode* pColNode = (SColumnNode*)pSortKey->pExpr; bi.slotId = pColNode->slotId; - // pColNode->order; - // SColumn c = {0}; - // c.slotId = pColNode->slotId; - // c.colId = pColNode->colId; - // c.type = pColNode->node.resType.type; - // c.bytes = pColNode->node.resType.bytes; - // c.precision = pColNode->node.resType.precision; - // c.scale = pColNode->node.resType.scale; - taosArrayPush(pList, &bi); } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index b3a8e09f166e05a490fb865e298cdf361473faa1..86c2ad4f21819367ff5ae2d25603db722a0e9d35 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -25,7 +25,11 @@ #include "thash.h" #include "ttypes.h" -static void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput) { +static int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity); +static void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInfo** pGroupInfo, int32_t len); +static uint64_t calcGroupId(char* pData, int32_t len); + +static void destroyGroupOperatorInfo(void* param, int32_t numOfOutput) { SGroupbyOperatorInfo* pInfo = (SGroupbyOperatorInfo*)param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); taosMemoryFreeClear(pInfo->keyBuf); @@ -33,44 +37,43 @@ static void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput) { taosArrayDestroy(pInfo->pGroupColVals); } -static int32_t initGroupOptrInfo(SGroupbyOperatorInfo* pInfo, SArray* pGroupColList) { - pInfo->pGroupColVals = taosArrayInit(4, sizeof(SGroupKeys)); - if (pInfo->pGroupColVals == NULL) { +static int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char** keyBuf, const SArray* pGroupColList) { + *pGroupColVals = taosArrayInit(4, sizeof(SGroupKeys)); + if ((*pGroupColVals) == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } int32_t numOfGroupCols = taosArrayGetSize(pGroupColList); for (int32_t i = 0; i < numOfGroupCols; ++i) { SColumn* pCol = taosArrayGet(pGroupColList, i); - pInfo->groupKeyLen += pCol->bytes; + (*keyLen) += pCol->bytes; struct SGroupKeys key = {0}; - key.bytes = pCol->bytes; - key.type = pCol->type; + key.bytes = pCol->bytes; + key.type = pCol->type; key.isNull = false; - key.pData = taosMemoryCalloc(1, pCol->bytes); + key.pData = taosMemoryCalloc(1, pCol->bytes); if (key.pData == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - taosArrayPush(pInfo->pGroupColVals, &key); + taosArrayPush((*pGroupColVals), &key); } int32_t nullFlagSize = sizeof(int8_t) * numOfGroupCols; - pInfo->keyBuf = taosMemoryCalloc(1, pInfo->groupKeyLen + nullFlagSize); - if (pInfo->keyBuf == NULL) { + (*keyBuf) = taosMemoryCalloc(1, (*keyLen) + nullFlagSize); + if ((*keyBuf) == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } return TSDB_CODE_SUCCESS; } -static bool groupKeyCompare(SGroupbyOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t rowIndex, - int32_t numOfGroupCols) { +static bool groupKeyCompare(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlock* pBlock, int32_t rowIndex, int32_t numOfGroupCols) { SColumnDataAgg* pColAgg = NULL; for (int32_t i = 0; i < numOfGroupCols; ++i) { - SColumn* pCol = taosArrayGet(pInfo->pGroupCols, i); + SColumn* pCol = taosArrayGet(pGroupCols, i); SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); if (pBlock->pBlockAgg != NULL) { pColAgg = &pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? @@ -78,7 +81,7 @@ static bool groupKeyCompare(SGroupbyOperatorInfo* pInfo, SSDataBlock* pBlock, in bool isNull = colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg); - SGroupKeys* pkey = taosArrayGet(pInfo->pGroupColVals, i); + SGroupKeys* pkey = taosArrayGet(pGroupColVals, i); if (pkey->isNull && isNull) { continue; } @@ -106,18 +109,18 @@ static bool groupKeyCompare(SGroupbyOperatorInfo* pInfo, SSDataBlock* pBlock, in return true; } -static void recordNewGroupKeys(SGroupbyOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t rowIndex, int32_t numOfGroupCols) { +static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlock* pBlock, int32_t rowIndex, int32_t numOfGroupCols) { SColumnDataAgg* pColAgg = NULL; for (int32_t i = 0; i < numOfGroupCols; ++i) { - SColumn* pCol = taosArrayGet(pInfo->pGroupCols, i); + SColumn* pCol = taosArrayGet(pGroupCols, i); SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); if (pBlock->pBlockAgg != NULL) { pColAgg = &pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? } - SGroupKeys* pkey = taosArrayGet(pInfo->pGroupColVals, i); + SGroupKeys* pkey = taosArrayGet(pGroupColVals, i); if (colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg)) { pkey->isNull = true; } else { @@ -197,13 +200,13 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { for (int32_t j = 0; j < pBlock->info.rows; ++j) { // Compare with the previous row of this column, and do not set the output buffer again if they are identical. if (!pInfo->isInit) { - recordNewGroupKeys(pInfo, pBlock, j, numOfGroupCols); + recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j, numOfGroupCols); pInfo->isInit = true; num++; continue; } - bool equal = groupKeyCompare(pInfo, pBlock, j, numOfGroupCols); + bool equal = groupKeyCompare(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j, numOfGroupCols); if (equal) { num++; continue; @@ -212,7 +215,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { // The first row of a new block does not belongs to the previous existed group if (!equal && j == 0) { num++; - recordNewGroupKeys(pInfo, pBlock, j, numOfGroupCols); + recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j, numOfGroupCols); continue; } @@ -227,7 +230,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { // assign the group keys or user input constant values if required doAssignGroupKeys(pCtx, pOperator->numOfOutput, pBlock->info.rows, rowIndex); - recordNewGroupKeys(pInfo, pBlock, j, numOfGroupCols); + recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j, numOfGroupCols); num = 1; } @@ -259,7 +262,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou if (pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } - return pRes; + return (pRes->info.rows == 0)? NULL:pRes; } int32_t order = TSDB_ORDER_ASC; @@ -309,7 +312,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou } } - return pInfo->binfo.pRes; + return (pRes->info.rows == 0)? NULL:pRes; } SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SArray* pGroupColList, SNode* pCondition, SExecTaskInfo* pTaskInfo, @@ -325,7 +328,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, 4096, pResultBlock, pTaskInfo->id.str); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - int32_t code = initGroupOptrInfo(pInfo, pGroupColList); + int32_t code = initGroupOptrInfo(&pInfo->pGroupColVals, &pInfo->groupKeyLen, &pInfo->keyBuf, pGroupColList); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -339,7 +342,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx pOperator->info = pInfo; pOperator->_openFn = operatorDummyOpenFn; pOperator->getNextFn = hashGroupbyAggregate; - pOperator->closeFn = destroyGroupbyOperatorInfo; + pOperator->closeFn = destroyGroupOperatorInfo; code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -351,67 +354,263 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx return NULL; } -static SSDataBlock* doPartitionData(SOperatorInfo* pOperator, bool* newgroup) { +static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { +// SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + SPartitionOperatorInfo* pInfo = pOperator->info; + + int32_t numOfGroupCols = taosArrayGetSize(pInfo->pGroupCols); + for (int32_t j = 0; j < pBlock->info.rows; ++j) { + recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j, numOfGroupCols); + int32_t len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals); + + SDataGroupInfo* pGInfo = NULL; + void *pPage = getCurrentDataGroupInfo(pInfo, &pGInfo, len); + + pGInfo->numOfRows += 1; + if (pGInfo->groupId == 0) { + pGInfo->groupId = calcGroupId(pInfo->keyBuf, len); + } + + int32_t* rows = (int32_t*) pPage; + + size_t numOfCols = pOperator->numOfOutput; + for(int32_t i = 0; i < numOfCols; ++i) { + SExprInfo* pExpr = &pOperator->pExpr[i]; + int32_t slotId = pExpr->base.pParam[0].pCol->slotId; + + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); + + int32_t bytes = pColInfoData->info.bytes; + int32_t startOffset = pInfo->columnOffset[i]; + + char* columnLen = NULL; + int32_t contentLen = 0; + + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + int32_t* offset = pPage + startOffset; + columnLen = pPage + startOffset + sizeof(int32_t) * pInfo->rowCapacity; + char* data = (char*)(columnLen + sizeof(int32_t)); + + if (colDataIsNull_s(pColInfoData, j)) { + offset[(*rows)] = -1; + contentLen = 0; + } else { + offset[*rows] = (*columnLen); + char* src = colDataGetData(pColInfoData, j); + memcpy(data + (*columnLen), src, varDataTLen(src)); + contentLen = varDataTLen(src); + } + } else { + char* bitmap = pPage + startOffset; + columnLen = pPage + startOffset + BitmapLen(pInfo->rowCapacity); + char* data = (char*) columnLen + sizeof(int32_t); + + bool isNull = colDataIsNull_f(pColInfoData->nullbitmap, j); + if (isNull) { + colDataSetNull_f(bitmap, (*rows)); + } else { + memcpy(data + (*columnLen), colDataGetData(pColInfoData, j), bytes); + } + contentLen = bytes; + } + + (*columnLen) += contentLen; + } + + (*rows) += 1; + + setBufPageDirty(pPage, true); + releaseBufPage(pInfo->pBuf, pPage); + } +} + +void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInfo** pGroupInfo, int32_t len) { + SDataGroupInfo* p = taosHashGet(pInfo->pGroupSet, pInfo->keyBuf, len); + + void* pPage = NULL; + if (p == NULL) { // it is a new group + SDataGroupInfo gi = {0}; + gi.pPageList = taosArrayInit(100, sizeof(int32_t)); + taosHashPut(pInfo->pGroupSet, pInfo->keyBuf, len, &gi, sizeof(SDataGroupInfo)); + + p = taosHashGet(pInfo->pGroupSet, pInfo->keyBuf, len); + + int32_t pageId = 0; + pPage = getNewBufPage(pInfo->pBuf, 0, &pageId); + taosArrayPush(p->pPageList, &pageId); + + *(int32_t *) pPage = 0; + } else { + int32_t* curId = taosArrayGetLast(p->pPageList); + pPage = getBufPage(pInfo->pBuf, *curId); + + int32_t *rows = (int32_t*) pPage; + if (*rows >= pInfo->rowCapacity) { + // add a new page for current group + int32_t pageId = 0; + pPage = getNewBufPage(pInfo->pBuf, 0, &pageId); + taosArrayPush(p->pPageList, &pageId); + + *(int32_t*) pPage = 0; + } + } + + *pGroupInfo = p; + return pPage; +} + +uint64_t calcGroupId(char* pData, int32_t len) { + T_MD5_CTX context; + tMD5Init(&context); + tMD5Update(&context, (uint8_t*)pData, len); + tMD5Final(&context); + + // NOTE: only extract the initial 8 bytes of the final MD5 digest + uint64_t id = 0; + memcpy(&id, context.digest, sizeof(uint64_t)); + return id; +} + +int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity) { + size_t numOfCols = pBlock->info.numOfCols; + int32_t* offset = taosMemoryCalloc(pBlock->info.numOfCols, sizeof(int32_t)); + + offset[0] = sizeof(int32_t); // the number of rows in current page, ref to SSDataBlock paged serialization format + + for(int32_t i = 0; i < numOfCols - 1; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + + int32_t bytes = pColInfoData->info.bytes; + int32_t payloadLen = bytes * rowCapacity; + + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + // offset segment + content length + payload + offset[i + 1] = rowCapacity * sizeof(int32_t) + sizeof(int32_t) + payloadLen + offset[i]; + } else { + // bitmap + content length + payload + offset[i + 1] = BitmapLen(rowCapacity) + sizeof(int32_t) + payloadLen + offset[i]; + } + } + + return offset; +} + +static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { + SPartitionOperatorInfo* pInfo = pOperator->info; + + SDataGroupInfo* pGroupInfo = pInfo->pGroupIter; + if (pInfo->pGroupIter == NULL || pInfo->pageIndex >= taosArrayGetSize(pGroupInfo->pPageList)) { + // try next group data + pInfo->pGroupIter = taosHashIterate(pInfo->pGroupSet, pInfo->pGroupIter); + if (pInfo->pGroupIter == NULL) { + pOperator->status = OP_EXEC_DONE; + return NULL; + } + + pGroupInfo = pInfo->pGroupIter; + pInfo->pageIndex = 0; + } + + int32_t* pageId = taosArrayGet(pGroupInfo->pPageList, pInfo->pageIndex); + void* page = getBufPage(pInfo->pBuf, *pageId); + + blockDataFromBuf1(pInfo->binfo.pRes, page, pInfo->rowCapacity); + + pInfo->pageIndex += 1; + + pInfo->binfo.pRes->info.groupId = pGroupInfo->groupId; + return pInfo->binfo.pRes; +} + +static SSDataBlock* hashPartition(SOperatorInfo* pOperator, bool* newgroup) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SSortOperatorInfo* pInfo = pOperator->info; - bool hasVarCol = pInfo->pDataBlock->info.hasVarCol; + SGroupbyOperatorInfo* pInfo = pOperator->info; + SSDataBlock* pRes = pInfo->binfo.pRes; if (pOperator->status == OP_RES_TO_RETURN) { - return getSortedBlockData(pInfo->pSortHandle, pInfo->pDataBlock, pInfo->numOfRowsInRes); + blockDataCleanup(pRes); + return buildPartitionResult(pOperator); } - int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, pInfo->bufPageSize, numOfBufPage, - pInfo->pDataBlock, pTaskInfo->id.str); - - tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock); + SOperatorInfo* downstream = pOperator->pDownstream[0]; - SGenericSource* ps = taosMemoryCalloc(1, sizeof(SGenericSource)); - ps->param = pOperator->pDownstream[0]; - tsortAddSource(pInfo->pSortHandle, ps); + while (1) { + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + if (pBlock == NULL) { + break; + } - int32_t code = tsortOpen(pInfo->pSortHandle); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, terrno); + // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->binfo.pCtx, pOperator->numOfOutput); + doHashPartition(pOperator, pBlock); } pOperator->status = OP_RES_TO_RETURN; - return getSortedBlockData(pInfo->pSortHandle, pInfo->pDataBlock, pInfo->numOfRowsInRes); + blockDataEnsureCapacity(pRes, 4096); + return buildPartitionResult(pOperator); } -SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResultBlock, SArray* pSortInfo, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) { - SSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); +static void destroyPartitionOperatorInfo(void* param, int32_t numOfOutput) { + SPartitionOperatorInfo* pInfo = (SPartitionOperatorInfo*)param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); + taosArrayDestroy(pInfo->pGroupCols); + taosArrayDestroy(pInfo->pGroupColVals); + taosMemoryFree(pInfo->keyBuf); + taosMemoryFree(pInfo->columnOffset); +} + +SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SArray* pGroupColList, + SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) { + SPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SPartitionOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } - pInfo->sortBufSize = 1024 * 16; // TODO dynamic set the available sort buffer - pInfo->bufPageSize = 1024; - pInfo->numOfRowsInRes = 1024; - pInfo->pDataBlock = pResultBlock; - pInfo->pSortInfo = pSortInfo; + pInfo->pGroupCols = pGroupColList; + + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pGroupSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK); + if (pInfo->pGroupSet == NULL) { + goto _error; + } + + int32_t code = createDiskbasedBuf(&pInfo->pBuf, 4096, 4096 * 256, pTaskInfo->id.str, "/tmp/"); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + pInfo->rowCapacity = blockDataGetCapacityInRow(pResultBlock, getBufPageSize(pInfo->pBuf)); + pInfo->columnOffset = setupColumnOffset(pResultBlock, pInfo->rowCapacity); + code = initGroupOptrInfo(&pInfo->pGroupColVals, &pInfo->groupKeyLen, &pInfo->keyBuf, pGroupColList); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } pOperator->name = "PartitionOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PARTITION; pOperator->blockingOptr = true; pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PARTITION; - pOperator->pTaskInfo = pTaskInfo; - pOperator->getNextFn = doPartitionData; -// pOperator->closeFn = destroyOrderOperatorInfo; + pInfo->binfo.pRes = pResultBlock; + pOperator->numOfOutput = numOfCols; + pOperator->pExpr = pExprInfo; + pOperator->info = pInfo; + pOperator->_openFn = operatorDummyOpenFn; + pOperator->getNextFn = hashPartition; + pOperator->closeFn = destroyPartitionOperatorInfo; - int32_t code = appendDownstream(pOperator, &downstream, 1); + code = appendDownstream(pOperator, &downstream, 1); return pOperator; _error: pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(pInfo); - taosMemoryFree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); return NULL; } \ No newline at end of file diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 53fe723671d0b5a68047109f899fe98656e592ca..c67f8336994d010694a4b9b42d44ff7854623c11 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -28,7 +28,7 @@ #include "query.h" #include "tcompare.h" #include "thash.h" -#include "tsdb.h" +#include "vnode.h" #include "ttypes.h" #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 7a57d6296912482d913edb61170d9d1d04780e1b..5dfda92982d9643ad3c397aefd4018722e6d4d84 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -425,9 +425,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { qDebug("%s %d rounds mergesort required to complete the sort, first-round sorted data size:%"PRIzu", sort:%"PRId64", total elapsed:%"PRId64, pHandle->idStr, (int32_t) (sortPass + 1), getTotalBufSize(pHandle->pBuf), pHandle->sortElapsed, pHandle->totalElapsed); - size_t pgSize = pHandle->pageSize; - int32_t numOfRows = (pgSize - blockDataGetSerialMetaSize(pHandle->pDataBlock))/ blockDataGetSerialRowSize(pHandle->pDataBlock); - + int32_t numOfRows = blockDataGetCapacityInRow(pHandle->pDataBlock, pHandle->pageSize); blockDataEnsureCapacity(pHandle->pDataBlock, numOfRows); size_t numOfSorted = taosArrayGetSize(pHandle->pOrderedSource); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index c3bc696a534cdc84a6dfc0314fd70ab3b59cc6ac..8f90670d13f48656f0a24c3f68300e55063aa5de 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1578,7 +1578,7 @@ static int32_t datumToJson(const void* pObj, SJson* pJson) { case TSDB_DATA_TYPE_NULL: break; case TSDB_DATA_TYPE_BOOL: - code = tjsonAddIntegerToObject(pJson, jkValueDatum, pNode->datum.b); + code = tjsonAddBoolToObject(pJson, jkValueDatum, pNode->datum.b); break; case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 95a3280fc2f9d4e24ceece2005f9b0ac24b661f8..074ec4dcae542b0b941d5f37a801e69c917e2a77 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -644,7 +644,7 @@ SNodeList* nodesMakeList() { int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode) { if (NULL == pList || NULL == pNode) { - return TSDB_CODE_SUCCESS; + return TSDB_CODE_FAILED; } SListCell* p = taosMemoryCalloc(1, sizeof(SListCell)); if (NULL == p) { @@ -688,7 +688,7 @@ int32_t nodesListMakeAppend(SNodeList** pList, SNodeptr pNode) { int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc) { if (NULL == pTarget || NULL == pSrc) { - return TSDB_CODE_SUCCESS; + return TSDB_CODE_FAILED; } if (NULL == pTarget->pHead) { @@ -717,11 +717,34 @@ int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc) { return code; } +int32_t nodesListPushFront(SNodeList* pList, SNodeptr pNode) { + if (NULL == pList || NULL == pNode) { + return TSDB_CODE_FAILED; + } + SListCell* p = taosMemoryCalloc(1, sizeof(SListCell)); + if (NULL == p) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; + } + p->pNode = pNode; + if (NULL != pList->pHead) { + pList->pHead->pPrev = p; + p->pNext = pList->pHead; + } + pList->pHead = p; + ++(pList->length); + return TSDB_CODE_SUCCESS; +} + SListCell* nodesListErase(SNodeList* pList, SListCell* pCell) { if (NULL == pCell->pPrev) { pList->pHead = pCell->pNext; } else { pCell->pPrev->pNext = pCell->pNext; + } + if (NULL == pCell->pNext) { + pList->pTail = pCell->pPrev; + } else { pCell->pNext->pPrev = pCell->pPrev; } SListCell* pNext = pCell->pNext; @@ -731,6 +754,24 @@ SListCell* nodesListErase(SNodeList* pList, SListCell* pCell) { return pNext; } +void nodesListInsertList(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc) { + if (NULL == pTarget || NULL == pPos || NULL == pSrc) { + return; + } + + if (NULL == pPos->pPrev) { + pTarget->pHead = pSrc->pHead; + } else { + pPos->pPrev->pNext = pSrc->pHead; + } + pSrc->pHead->pPrev = pPos->pPrev; + pSrc->pTail->pNext = pPos; + pPos->pPrev = pSrc->pTail; + + pTarget->length += pSrc->length; + taosMemoryFreeClear(pSrc); +} + SNodeptr nodesListGetNode(SNodeList* pList, int32_t index) { SNode* node; FOREACH(node, pList) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 8b3bf5b4925f1bf442b5883ff59634667c43dfb0..ebe5b639c444b82a20a1725f59e7b96761955fa0 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -376,8 +376,9 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode* pCol) { } static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { + uint8_t precision = (NULL != pCxt->pCurrStmt ? pCxt->pCurrStmt->precision : pVal->node.resType.precision); if (pVal->isDuration) { - if (parseNatualDuration(pVal->literal, strlen(pVal->literal), &pVal->datum.i, &pVal->unit, pVal->node.resType.precision) != TSDB_CODE_SUCCESS) { + if (parseNatualDuration(pVal->literal, strlen(pVal->literal), &pVal->datum.i, &pVal->unit, precision) != TSDB_CODE_SUCCESS) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } } else { @@ -421,7 +422,7 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { break; } case TSDB_DATA_TYPE_TIMESTAMP: { - if (taosParseTime(pVal->literal, &pVal->datum.i, pVal->node.resType.bytes, pVal->node.resType.precision, tsDaylight) != TSDB_CODE_SUCCESS) { + if (taosParseTime(pVal->literal, &pVal->datum.i, pVal->node.resType.bytes, precision, tsDaylight) != TSDB_CODE_SUCCESS) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } break; @@ -683,6 +684,19 @@ static int32_t setTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTa return code; } +static uint8_t getStmtPrecision(SNode* pStmt) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + return ((SSelectStmt*)pStmt)->precision; + } + return 0; +} + +static uint8_t getJoinTablePrecision(SJoinTableNode* pJoinTable) { + uint8_t lp = ((STableNode*)pJoinTable->pLeft)->precision; + uint8_t rp = ((STableNode*)pJoinTable->pRight)->precision; + return (lp > rp ? rp : lp); +} + static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pTable)) { @@ -695,6 +709,7 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { if (TSDB_CODE_SUCCESS != code) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TABLE_NOT_EXIST, pRealTable->table.tableName); } + pRealTable->table.precision = pRealTable->pMeta->tableInfo.precision; code = setTableVgroupList(pCxt, &name, pRealTable); if (TSDB_CODE_SUCCESS == code) { code = addNamespace(pCxt, pRealTable); @@ -705,6 +720,7 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { STempTableNode* pTempTable = (STempTableNode*)pTable; code = translateSubquery(pCxt, pTempTable->pSubquery); if (TSDB_CODE_SUCCESS == code) { + pTempTable->table.precision = getStmtPrecision(pTempTable->pSubquery); code = addNamespace(pCxt, pTempTable); } break; @@ -716,6 +732,7 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { code = translateTable(pCxt, pJoinTable->pRight); } if (TSDB_CODE_SUCCESS == code) { + pJoinTable->table.precision = getJoinTablePrecision(pJoinTable); code = translateExpr(pCxt, pJoinTable->pOnCond); } break; @@ -726,24 +743,99 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { return code; } -static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect, bool* pIsSelectStar) { - if (NULL == pSelect->pProjectionList) { // select * ... - SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); - size_t nums = taosArrayGetSize(pTables); - pSelect->pProjectionList = nodesMakeList(); - if (NULL == pSelect->pProjectionList) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_OUT_OF_MEMORY); - } - for (size_t i = 0; i < nums; ++i) { - STableNode* pTable = taosArrayGetP(pTables, i); - int32_t code = createColumnNodeByTable(pCxt, pTable, pSelect->pProjectionList); - if (TSDB_CODE_SUCCESS != code) { - return code; - } +static int32_t createAllColumns(STranslateContext* pCxt, SNodeList** pCols) { + *pCols = nodesMakeList(); + if (NULL == *pCols) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_OUT_OF_MEMORY); + } + SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); + size_t nums = taosArrayGetSize(pTables); + for (size_t i = 0; i < nums; ++i) { + STableNode* pTable = taosArrayGetP(pTables, i); + int32_t code = createColumnNodeByTable(pCxt, pTable, *pCols); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + } + return TSDB_CODE_SUCCESS; +} + +static bool isFirstLastFunc(SFunctionNode* pFunc) { + return (FUNCTION_TYPE_FIRST == pFunc->funcType || FUNCTION_TYPE_LAST == pFunc->funcType); +} + +static bool isFirstLastStar(SNode* pNode) { + if (QUERY_NODE_FUNCTION != nodeType(pNode) || !isFirstLastFunc((SFunctionNode*)pNode)) { + return false; + } + SNodeList* pParameterList = ((SFunctionNode*)pNode)->pParameterList; + if (LIST_LENGTH(pParameterList) != 1) { + return false; + } + SNode* pParam = nodesListGetNode(pParameterList, 0); + return (QUERY_NODE_COLUMN == nodeType(pParam) ? 0 == strcmp(((SColumnNode*)pParam)->colName, "*") : false); +} + +static SNode* createFirstLastFunc(SFunctionNode* pSrcFunc, SColumnNode* pCol) { + SFunctionNode* pFunc = nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFunc) { + return NULL; + } + pFunc->pParameterList = nodesMakeList(); + if (NULL == pFunc->pParameterList || TSDB_CODE_SUCCESS != nodesListAppend(pFunc->pParameterList, pCol)) { + nodesDestroyNode(pFunc); + return NULL; + } + + pFunc->node.resType = pCol->node.resType; + pFunc->funcId = pSrcFunc->funcId; + pFunc->funcType = pSrcFunc->funcType; + strcpy(pFunc->functionName, pSrcFunc->functionName); + snprintf(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), (FUNCTION_TYPE_FIRST == pSrcFunc->funcType ? "first(%s)" : "last(%s)"), pCol->colName); + + return (SNode*)pFunc; +} + +static int32_t createFirstLastAllCols(STranslateContext* pCxt, SFunctionNode* pSrcFunc, SNodeList** pOutput) { + SNodeList* pCols = NULL; + if (TSDB_CODE_SUCCESS != createAllColumns(pCxt, &pCols)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SNodeList* pFuncs = nodesMakeList(); + if (NULL == pFuncs) { + return TSDB_CODE_OUT_OF_MEMORY; + } + SNode* pCol = NULL; + FOREACH(pCol, pCols) { + if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pFuncs, createFirstLastFunc(pSrcFunc, (SColumnNode*)pCol))) { + nodesDestroyNode(pFuncs); + return TSDB_CODE_OUT_OF_MEMORY; } - *pIsSelectStar = true; + } + + *pOutput = pFuncs; + return TSDB_CODE_SUCCESS; +} + +static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect) { + if (NULL == pSelect->pProjectionList) { // select * ... + return createAllColumns(pCxt, &pSelect->pProjectionList); } else { // todo : t.* + SNode* pNode = NULL; + WHERE_EACH(pNode, pSelect->pProjectionList) { + if (isFirstLastStar(pNode)) { + SNodeList* pFuncs = NULL; + if (TSDB_CODE_SUCCESS != createFirstLastAllCols(pCxt, (SFunctionNode*)pNode, &pFuncs)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + INSERT_LIST(pSelect->pProjectionList, pFuncs); + ERASE_NODE(pSelect->pProjectionList); + continue; + } + WHERE_NEXT; + } } return TSDB_CODE_SUCCESS; } @@ -780,8 +872,8 @@ static int32_t getPositionValue(const SValueNode* pVal) { static int32_t translateOrderByPosition(STranslateContext* pCxt, SNodeList* pProjectionList, SNodeList* pOrderByList, bool* pOther) { *pOther = false; - SNode* pNode; - FOREACH(pNode, pOrderByList) { + SNode* pNode = NULL; + WHERE_EACH(pNode, pOrderByList) { SNode* pExpr = ((SOrderByExprNode*)pNode)->pExpr; if (QUERY_NODE_VALUE == nodeType(pExpr)) { SValueNode* pVal = (SValueNode*)pExpr; @@ -806,6 +898,7 @@ static int32_t translateOrderByPosition(STranslateContext* pCxt, SNodeList* pPro } else { *pOther = true; } + WHERE_NEXT; } return TSDB_CODE_SUCCESS; } @@ -828,11 +921,10 @@ static int32_t translateOrderBy(STranslateContext* pCxt, SSelectStmt* pSelect) { } static int32_t translateSelectList(STranslateContext* pCxt, SSelectStmt* pSelect) { - bool isSelectStar = false; - int32_t code = translateStar(pCxt, pSelect, &isSelectStar); - if (TSDB_CODE_SUCCESS == code && !isSelectStar) { - pCxt->currClause = SQL_CLAUSE_SELECT; - code = translateExprList(pCxt, pSelect->pProjectionList); + pCxt->currClause = SQL_CLAUSE_SELECT; + int32_t code = translateExprList(pCxt, pSelect->pProjectionList); + if (TSDB_CODE_SUCCESS == code) { + code = translateStar(pCxt, pSelect); } if (TSDB_CODE_SUCCESS == code) { code = checkExprListForGroupBy(pCxt, pSelect->pProjectionList); @@ -852,9 +944,12 @@ static int32_t translateHaving(STranslateContext* pCxt, SSelectStmt* pSelect) { return code; } -static int32_t translateGroupBy(STranslateContext* pCxt, SNodeList* pGroupByList) { +static int32_t translateGroupBy(STranslateContext* pCxt, SSelectStmt* pSelect) { + if (NULL != pSelect->pGroupByList && NULL != pSelect->pWindow) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GROUPBY_WINDOW_COEXIST); + } pCxt->currClause = SQL_CLAUSE_GROUP_BY; - return translateExprList(pCxt, pGroupByList); + return translateExprList(pCxt, pSelect->pGroupByList); } static int32_t translateIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* pInterval) { @@ -899,14 +994,18 @@ static int32_t translateWhere(STranslateContext* pCxt, SNode* pWhere) { return translateExpr(pCxt, pWhere); } -static int32_t translateFrom(STranslateContext* pCxt, SNode* pTable) { +static int32_t translateFrom(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->currClause = SQL_CLAUSE_FROM; - return translateTable(pCxt, pTable); + int32_t code = translateTable(pCxt, pSelect->pFromTable); + if (TSDB_CODE_SUCCESS == code) { + pSelect->precision = ((STableNode*)pSelect->pFromTable)->precision; + } + return code; } static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->pCurrStmt = pSelect; - int32_t code = translateFrom(pCxt, pSelect->pFromTable); + int32_t code = translateFrom(pCxt, pSelect); if (TSDB_CODE_SUCCESS == code) { code = translateWhere(pCxt, pSelect->pWhere); } @@ -917,7 +1016,7 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { code = translateWindow(pCxt, pSelect->pWindow); } if (TSDB_CODE_SUCCESS == code) { - code = translateGroupBy(pCxt, pSelect->pGroupByList); + code = translateGroupBy(pCxt, pSelect); } if (TSDB_CODE_SUCCESS == code) { code = translateHaving(pCxt, pSelect); @@ -1807,10 +1906,13 @@ static int32_t getSmaIndexBuildAst(STranslateContext* pCxt, SCreateIndexStmt* pS pSelect->pFromTable = (SNode*)pTable; pSelect->pProjectionList = nodesCloneList(pStmt->pOptions->pFuncs); - if (NULL == pTable) { + SFunctionNode* pFunc = nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pSelect->pProjectionList || NULL == pFunc) { nodesDestroyNode(pSelect); return TSDB_CODE_OUT_OF_MEMORY; } + strcpy(pFunc->functionName, "_wstartts"); + nodesListPushFront(pSelect->pProjectionList, pFunc); SNode* pProject = NULL; FOREACH(pProject, pSelect->pProjectionList) { sprintf(((SExprNode*)pProject)->aliasName, "#sma_%p", pProject); diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 5206e0f8a52ca410b991dcaf2f7894fc163d5616..171324aa99680b2daa6456af8e66a0204d405aeb 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -89,6 +89,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "invalid option rollup: only one function is allowed"; case TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION: return "invalid option retentions"; + case TSDB_CODE_PAR_GROUPBY_WINDOW_COEXIST: + return "GROUP BY and WINDOW-clause can't be used together"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 1a6abb7100a304026628ca5aad6c194820744064..7ce7b422aca5c216b9119639f910e166edb406e3 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -267,7 +267,7 @@ static void trtrim(char *input, char *output, int32_t type, int32_t charLen) { numOfSpaces++; } } else { //NCHAR - for (int32_t i = charLen - 1; i < charLen; ++i) { + for (int32_t i = charLen - 1; i >= 0; --i) { if (!iswspace(*((uint32_t *)varDataVal(input) + i))) { break; } @@ -295,11 +295,11 @@ static int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarP SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; - char *in = pInputData->pData; + char *in = pInputData->pData + pInputData->varmeta.offset[0]; int16_t *out = (int16_t *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { - if (colDataIsNull_f(pInputData->nullbitmap, i)) { + if (colDataIsNull_s(pInputData, i)) { colDataSetNull_f(pOutputData->nullbitmap, i); continue; } @@ -357,7 +357,7 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu } for (int32_t i = 0; i < inputNum; ++i) { pInputData[i] = pInput[i].columnData; - input[i] = pInputData[i]->pData; + input[i] = pInputData[i]->pData + pInputData[i]->varmeta.offset[0]; int32_t factor = 1; if (hasNcharCol && (GET_PARAM_TYPE(&pInput[i]) == TSDB_DATA_TYPE_VARCHAR)) { factor = TSDB_NCHAR_SIZE; @@ -438,7 +438,7 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p } for (int32_t i = 0; i < inputNum; ++i) { pInputData[i] = pInput[i].columnData; - input[i] = pInputData[i]->pData; + input[i] = pInputData[i]->pData + pInputData[i]->varmeta.offset[0]; int32_t factor = 1; if (hasNcharCol && (GET_PARAM_TYPE(&pInput[i]) == TSDB_DATA_TYPE_VARCHAR)) { factor = TSDB_NCHAR_SIZE; @@ -509,7 +509,7 @@ static int32_t doCaseConvFunction(SScalarParam *pInput, int32_t inputNum, SScala SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; - char *input = pInputData->pData; + char *input = pInputData->pData + pInputData->varmeta.offset[0]; char *output = NULL; int32_t outputLen = pInputData->varmeta.length; @@ -554,7 +554,7 @@ static int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarPar SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; - char *input = pInputData->pData; + char *input = pInputData->pData + pInputData->varmeta.offset[0]; char *output = NULL; int32_t outputLen = pInputData->varmeta.length; @@ -606,7 +606,7 @@ int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; - char *input = pInputData->pData; + char *input = pInputData->pData + pInputData->varmeta.offset[0]; char *output = NULL; int32_t outputLen = pInputData->varmeta.length * pInput->numOfRows; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index d633284e13c19c68afd53169d7f3c07663d9b375..0ab2f4e010a3c738895bbad34ef95db8efda28bd 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -205,10 +205,10 @@ static void destroyThrdObj(SCliThrdObj* pThrd); transRefCliHandle(conn); \ } \ } while (0) -#define CONN_NO_PERSIST_BY_APP(conn) \ - (((conn)->status == ConnNormal || (conn)->status == ConnInPool) && T_REF_VAL_GET(conn) == 1) -#define CONN_RELEASE_BY_SERVER(conn) \ - (((conn)->status == ConnRelease || (conn)->status == ConnInPool) && T_REF_VAL_GET(conn) == 1) + +#define CONN_NO_PERSIST_BY_APP(conn) (((conn)->status == ConnNormal || (conn)->status == ConnInPool) && T_REF_VAL_GET(conn) == 1) +#define CONN_RELEASE_BY_SERVER(conn) (((conn)->status == ConnRelease || (conn)->status == ConnInPool) && T_REF_VAL_GET(conn) == 1) + #define REQUEST_NO_RESP(msg) ((msg)->noResp == 1) #define REQUEST_PERSIS_HANDLE(msg) ((msg)->persistHandle == 1) #define REQUEST_RELEASE_HANDLE(cmsg) ((cmsg)->type == Release) @@ -281,9 +281,8 @@ void cliHandleResp(SCliConn* conn) { tDebug("%s cli conn %p ref by app", CONN_GET_INST_LABEL(conn), conn); } - tDebug("%s cli conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pTransInst->label, conn, - TMSG_INFO(pHead->msgType), taosInetNtoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), - taosInetNtoa(conn->locaddr.sin_addr), ntohs(conn->locaddr.sin_port), transMsg.contLen); + tDebug("%s cli conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pTransInst->label, conn, TMSG_INFO(pHead->msgType), + taosInetNtoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), taosInetNtoa(conn->locaddr.sin_addr), ntohs(conn->locaddr.sin_port), transMsg.contLen); conn->secured = pHead->secured; @@ -349,12 +348,10 @@ void cliHandleExcept(SCliConn* pConn) { if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(pConn)) { transMsg.ahandle = transCtxDumpVal(&pConn->ctx, transMsg.msgType); - tDebug("%s cli conn %p construct ahandle %p by %s", CONN_GET_INST_LABEL(pConn), pConn, transMsg.ahandle, - TMSG_INFO(transMsg.msgType)); + tDebug("%s cli conn %p construct ahandle %p by %s", CONN_GET_INST_LABEL(pConn), pConn, transMsg.ahandle, TMSG_INFO(transMsg.msgType)); if (transMsg.ahandle == NULL) { transMsg.ahandle = transCtxDumpBrokenlinkVal(&pConn->ctx, (int32_t*)&(transMsg.msgType)); - tDebug("%s cli conn %p construct ahandle %p due to brokenlink", CONN_GET_INST_LABEL(pConn), pConn, - transMsg.ahandle); + tDebug("%s cli conn %p construct ahandle %p due to brokenlink", CONN_GET_INST_LABEL(pConn), pConn, transMsg.ahandle); } } else { transMsg.ahandle = pCtx ? pCtx->ahandle : NULL; @@ -628,9 +625,8 @@ void cliSend(SCliConn* pConn) { pHead->release = REQUEST_RELEASE_HANDLE(pCliMsg) ? 1 : 0; uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); - tDebug("%s cli conn %p %s is send to %s:%d, local info %s:%d", CONN_GET_INST_LABEL(pConn), pConn, - TMSG_INFO(pHead->msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), - taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); + tDebug("%s cli conn %p %s is send to %s:%d, local info %s:%d", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pHead->msgType), + taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); if (pHead->persist == 1) { CONN_SET_PERSIST_BY_APP(pConn); @@ -741,8 +737,13 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { if (ret) { tError("%s cli conn %p failed to set conn option, errmsg %s", pTransInst->label, conn, uv_err_name(ret)); } + struct sockaddr_in addr; - uv_ip4_addr(pMsg->ctx->ip, pMsg->ctx->port, &addr); + + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = taosGetIpv4FromFqdn(conn->ip); + addr.sin_port = (uint16_t)htons((uint16_t)conn->port); + // uv_ip4_addr(pMsg->ctx->ip, pMsg->ctx->port, &addr); // handle error in callback if fail to connect tTrace("%s cli conn %p try to connect to %s:%d", pTransInst->label, conn, pMsg->ctx->ip, pMsg->ctx->port); uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, cliConnCb); @@ -944,8 +945,8 @@ void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* p SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[index]; - tDebug("send request at thread:%d %p, dst: %s:%d", index, pMsg, ip, port); - transSendAsync(thrd->asyncPool, &(cliMsg->q)); + tDebug("send request at thread:%d %p, dst: %s:%d, app:%p", index, pMsg, ip, port, pMsg->ahandle); + ASSERT(transSendAsync(thrd->asyncPool, &(cliMsg->q)) == 0); } void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pReq, STransMsg* pRsp) { diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index d834263b940ae6932bcd121b5a18791a491012d5..84a2efb46c203e6dab2283093779ce4179fd736e 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -48,10 +48,8 @@ struct SDiskbasedBuf { }; static int32_t createDiskFile(SDiskbasedBuf* pBuf) { - // pBuf->file = fopen(pBuf->path, "wb+"); - pBuf->pFile = taosOpenFile(pBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); + pBuf->pFile = taosOpenFile(pBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL); if (pBuf->pFile == NULL) { - // qError("failed to create tmp file: %s on disk. %s", pBuf->path, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } diff --git a/tests/script/tsim/query/charScalarFunction.sim b/tests/script/tsim/query/charScalarFunction.sim index d7b1139b5ba714c0f3be467837588b41fbb73885..13108fecb7a651972f02ecd566721c67b4c8a0a3 100644 --- a/tests/script/tsim/query/charScalarFunction.sim +++ b/tests/script/tsim/query/charScalarFunction.sim @@ -286,25 +286,25 @@ endi #sql_error select c1, ltrim(t1), c2, ltrim(t2) from ctb3 -#print ====> rtrim -#sql select c1, rtrim(c1), c2, rtrim(c2), rtrim(" abcdEFGH =-*&% ") from ntb3 -#print ====> select c1, rtrim(c1), c2, rtrim(c2), rtrim(" abcdEFGH =-*&% ") from ctb3 -#sql select c1, rtrim(c1), c2, rtrim(c2), rtrim(" abcdEFGH =-*&% ") from ctb3 -#print ====> rows: $rows -#print ====> [ $data00 ] [ $data01 ] [ $data02 ] [ $data03 ] [ $data04 ] [ $data05 ] [ $data06 ] -#print ====> $data10 $data11 $data12 $data13 $data14 $data15 -#if $rows != 1 then -# return -1 -#endi -#if $data01 != @ abcd 1234@ then -# return -1 -#endi -#if $data03 != @ abcd 1234@ then -# return -1 -#endi -#if $data04 != @ abcdEFGH =-*&%@ then -# return -1 -#endi +print ====> rtrim +sql select c1, rtrim(c1), c2, rtrim(c2), rtrim(" abcdEFGH =-*&% ") from ntb3 +print ====> select c1, rtrim(c1), c2, rtrim(c2), rtrim(" abcdEFGH =-*&% ") from ctb3 +sql select c1, rtrim(c1), c2, rtrim(c2), rtrim(" abcdEFGH =-*&% ") from ctb3 +print ====> rows: $rows +print ====> [ $data00 ] [ $data01 ] [ $data02 ] [ $data03 ] [ $data04 ] [ $data05 ] [ $data06 ] +print ====> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 1 then + return -1 +endi +if $data01 != @ abcd 1234@ then + return -1 +endi +if $data03 != @ abcd 1234@ then + return -1 +endi +if $data04 != @ abcdEFGH =-*&%@ then + return -1 +endi #sql_error select c1, rtrim(t1), c2, rtrim(t2) from ctb3 diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 22d8c2b735a604e7b3f0a6c95bcd1db7a39b6ed5..7372745cb8788da15428fc40795f66fd0c66a6d1 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -274,17 +274,11 @@ int main(int32_t argc, char *argv[]) { loop_consume(tmq); - err = tmq_unsubscribe(tmq); - ASSERT(err == TMQ_RESP_ERR__SUCCESS); - - - #if 0 err = tmq_unsubscribe(tmq); if (err) { printf("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); exit(-1); } - #endif return 0; }