From 0d60f24db59ddcbb2f9d0701ad08c15d127361db Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 22 Nov 2022 22:42:44 +0800 Subject: [PATCH] refactor: do some internal refactor. --- source/common/src/tdatablock.c | 2 - source/libs/executor/CMakeLists.txt | 4 - source/libs/executor/inc/executorimpl.h | 146 +- source/libs/executor/src/cachescanoperator.c | 20 +- source/libs/executor/src/exchangeoperator.c | 88 + source/libs/executor/src/executorimpl.c | 192 -- source/libs/executor/src/groupoperator.c | 30 + source/libs/executor/src/joinoperator.c | 15 + source/libs/executor/src/projectoperator.c | 18 + source/libs/executor/src/scanoperator.c | 1710 +---------------- source/libs/executor/src/sortoperator.c | 12 + source/libs/executor/src/sysscanoperator.c | 1774 ++++++++++++++++++ 12 files changed, 1964 insertions(+), 2047 deletions(-) create mode 100644 source/libs/executor/src/sysscanoperator.c diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 3c2a5377e3..b5fffea73d 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1084,8 +1084,6 @@ int32_t dataBlockCompar_rv(const void* p1, const void* p2, const void* param) { return 0; } -int32_t varColSort(SColumnInfoData* pColumnInfoData, SBlockOrderInfo* pOrder) { return 0; } - int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst) { // Allocate the additional buffer. int64_t p0 = taosGetTimestampUs(); diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index 89d08b3078..8b3d04e32c 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -2,10 +2,6 @@ aux_source_directory(src EXECUTOR_SRC) #add_library(executor ${EXECUTOR_SRC}) add_library(executor STATIC ${EXECUTOR_SRC}) -#set_target_properties(executor PROPERTIES -# IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/libexecutor.a" -# INTERFACE_INCLUDE_DIRECTORIES "${TD_SOURCE_DIR}/include/libs/executor" -# ) target_link_libraries(executor PRIVATE os util common function parser planner qcom vnode scalar nodes index stream diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index f179c7bd41..f85bddcc31 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -235,16 +235,6 @@ typedef enum { #define COL_MATCH_FROM_COL_ID 0x1 #define COL_MATCH_FROM_SLOT_ID 0x2 -typedef struct SSourceDataInfo { - int32_t index; - SRetrieveTableRsp* pRsp; - uint64_t totalRows; - int64_t startTime; - int32_t code; - EX_SOURCE_STATUS status; - const char* taskId; -} SSourceDataInfo; - typedef struct SLoadRemoteDataInfo { uint64_t totalSize; // total load bytes from remote uint64_t totalRows; // total number of rows @@ -371,23 +361,8 @@ typedef struct STagScanInfo { SColMatchInfo matchInfo; int32_t curPos; SReadHandle readHandle; - STableListInfo* pTableList; } STagScanInfo; -typedef struct SLastrowScanInfo { - SSDataBlock* pRes; - SReadHandle readHandle; - void* pLastrowReader; - SColMatchInfo matchInfo; - int32_t* pSlotIds; - SExprSupp pseudoExprSup; - int32_t retrieveType; - int32_t currentGroupIndex; - SSDataBlock* pBufferredRes; - SArray* pUidList; - int32_t indexOfBufferedRes; -} SLastrowScanInfo; - typedef enum EStreamScanMode { STREAM_SCAN_FROM_READERHANDLE = 1, STREAM_SCAN_FROM_RES, @@ -504,40 +479,6 @@ typedef struct { SSnapContext* sContext; } SStreamRawScanInfo; -typedef struct SSysTableIndex { - int8_t init; - SArray* uids; - int32_t lastIdx; -} SSysTableIndex; - -typedef struct SSysTableScanInfo { - SRetrieveMetaTableRsp* pRsp; - SRetrieveTableReq req; - SEpSet epSet; - tsem_t ready; - SReadHandle readHandle; - int32_t accountId; - const char* pUser; - bool sysInfo; - bool showRewrite; - SNode* pCondition; // db_name filter condition, to discard data that are not in current database - SMTbCursor* pCur; // cursor for iterate the local table meta store. - SSysTableIndex* pIdx; // idx for local table meta - SColMatchInfo matchInfo; - SName name; - SSDataBlock* pRes; - int64_t numOfBlocks; // extract basic running information. - SLoadRemoteDataInfo loadInfo; -} SSysTableScanInfo; - -typedef struct SBlockDistInfo { - SSDataBlock* pResBlock; - STsdbReader* pHandle; - SReadHandle readHandle; - uint64_t uid; // table uid -} SBlockDistInfo; - -// todo remove this typedef struct SOptrBasicInfo { SResultRowInfo resultRowInfo; SSDataBlock* pRes; @@ -603,24 +544,6 @@ typedef struct SAggOperatorInfo { SExprSupp scalarExprSup; } SAggOperatorInfo; -typedef struct SProjectOperatorInfo { - SOptrBasicInfo binfo; - SAggSupporter aggSup; - SArray* pPseudoColInfo; - SLimitInfo limitInfo; - bool mergeDataBlocks; - SSDataBlock* pFinalRes; -} SProjectOperatorInfo; - -typedef struct SIndefOperatorInfo { - SOptrBasicInfo binfo; - SAggSupporter aggSup; - SArray* pPseudoColInfo; - SExprSupp scalarSup; - uint64_t groupId; - SSDataBlock* pNextGroupRes; -} SIndefOperatorInfo; - typedef struct SFillOperatorInfo { struct SFillInfo* pFillInfo; SSDataBlock* pRes; @@ -638,42 +561,12 @@ typedef struct SFillOperatorInfo { SExprSupp noFillExprSupp; } SFillOperatorInfo; -typedef struct SGroupbyOperatorInfo { - SOptrBasicInfo binfo; - SAggSupporter aggSup; - SArray* pGroupCols; // group by columns, SArray - SArray* pGroupColVals; // current group column values, SArray - bool isInit; // denote if current val is initialized or not - char* keyBuf; // group by keys for hash - int32_t groupKeyLen; // total group by column width - SGroupResInfo groupResInfo; - SExprSupp scalarSup; -} SGroupbyOperatorInfo; - typedef struct SDataGroupInfo { uint64_t groupId; int64_t numOfRows; SArray* pPageList; } SDataGroupInfo; -// The sort in partition may be needed later. -typedef struct SPartitionOperatorInfo { - SOptrBasicInfo binfo; - SArray* pGroupCols; - SArray* pGroupColVals; // current group column values, SArray - char* keyBuf; // group by keys for hash - int32_t groupKeyLen; // total group by column width - SHashObj* pGroupSet; // quick locate the window object for each result - - SDiskbasedBuf* pBuf; // query result buffer based on blocked-wised disk file - int32_t rowCapacity; // maximum number of rows for each buffer page - int32_t* columnOffset; // start position for each column data - SArray* sortedGroupArray; // SDataGroupInfo sorted by group id - int32_t groupIndex; // group index - int32_t pageIndex; // page index of current group - SExprSupp scalarSup; -} SPartitionOperatorInfo; - typedef struct SWindowRowsSup { STimeWindow win; TSKEY prevTs; @@ -800,33 +693,6 @@ typedef struct SStateWindowOperatorInfo { STimeWindowAggSupp twAggSup; } SStateWindowOperatorInfo; -typedef struct SSortOperatorInfo { - SOptrBasicInfo binfo; - uint32_t sortBufSize; // max buffer size for in-memory sort - SArray* pSortInfo; - SSortHandle* pSortHandle; - SColMatchInfo matchInfo; - int32_t bufPageSize; - int64_t startTs; // sort start time - uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included. - SLimitInfo limitInfo; -} SSortOperatorInfo; - -typedef struct SJoinOperatorInfo { - SSDataBlock* pRes; - int32_t joinType; - int32_t inputOrder; - - SSDataBlock* pLeft; - int32_t leftPos; - SColumnInfo leftCol; - - SSDataBlock* pRight; - int32_t rightPos; - SColumnInfo rightCol; - SNode* pCondAfterMerge; -} SJoinOperatorInfo; - #define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED) #define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED) @@ -850,7 +716,6 @@ void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGr void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf); -int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf); bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo); void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimitInfo); void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo, SOperatorInfo* pOperator); @@ -880,9 +745,6 @@ void cleanupAggSup(SAggSupporter* pAggSup); void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, const char* name); -int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts); -int32_t doGetScanStatus(SOperatorInfo* pOperator, uint64_t* uid, int64_t* ts); - SSDataBlock* loadNextDataBlock(void* param); void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset); @@ -965,9 +827,8 @@ void setInputDataBlock(SExprSupp* pExprSupp, SSDataBlock* pBlock, int32_t order, bool isTaskKilled(SExecTaskInfo* pTaskInfo); int32_t checkForQueryBuf(size_t numOfTables); -void setTaskKilled(SExecTaskInfo* pTaskInfo); -void queryCostStatis(SExecTaskInfo* pTaskInfo); - +void setTaskKilled(SExecTaskInfo* pTaskInfo); +void queryCostStatis(SExecTaskInfo* pTaskInfo); void doDestroyTask(SExecTaskInfo* pTaskInfo); void destroyOperatorInfo(SOperatorInfo* pOperator); int32_t getMaximumIdleDurationSec(); @@ -995,9 +856,6 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle); int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList); -int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result); -int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* length); - STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, int32_t order); int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey, diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 5e5c01201f..6b5f773fe3 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -25,6 +25,20 @@ #include "thash.h" #include "ttypes.h" +typedef struct SCacheRowsScanInfo { + SSDataBlock* pRes; + SReadHandle readHandle; + void* pLastrowReader; + SColMatchInfo matchInfo; + int32_t* pSlotIds; + SExprSupp pseudoExprSup; + int32_t retrieveType; + int32_t currentGroupIndex; + SSDataBlock* pBufferredRes; + SArray* pUidList; + int32_t indexOfBufferedRes; +} SCacheRowsScanInfo; + static SSDataBlock* doScanCache(SOperatorInfo* pOperator); static void destroyCacheScanOperator(void* param); static int32_t extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTaskInfo, int32_t** pSlotIds); @@ -33,7 +47,7 @@ static int32_t removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColM SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo) { int32_t code = TSDB_CODE_SUCCESS; - SLastrowScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SLastrowScanInfo)); + SCacheRowsScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SCacheRowsScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -114,7 +128,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { return NULL; } - SLastrowScanInfo* pInfo = pOperator->info; + SCacheRowsScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; STableListInfo* pTableList = pTaskInfo->pTableInfoList; @@ -240,7 +254,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } void destroyCacheScanOperator(void* param) { - SLastrowScanInfo* pInfo = (SLastrowScanInfo*)param; + SCacheRowsScanInfo* pInfo = (SCacheRowsScanInfo*)param; blockDataDestroy(pInfo->pRes); blockDataDestroy(pInfo->pBufferredRes); taosMemoryFree(pInfo->pSlotIds); diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index a28066003a..c858536bb1 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -41,6 +41,16 @@ typedef struct SFetchRspHandleWrapper { int32_t sourceIndex; } SFetchRspHandleWrapper; +typedef struct SSourceDataInfo { + int32_t index; + SRetrieveTableRsp* pRsp; + uint64_t totalRows; + int64_t startTime; + int32_t code; + EX_SOURCE_STATUS status; + const char* taskId; +} SSourceDataInfo; + static void destroyExchangeOperatorInfo(void* param); static void freeBlock(void* pParam); static void freeSourceDataInfo(void* param); @@ -52,6 +62,7 @@ static int32_t getCompletedSources(const SArray* pArray); static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator); static int32_t seqLoadRemoteData(SOperatorInfo* pOperator); static int32_t prepareLoadRemoteData(SOperatorInfo* pOperator); +static int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf); static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo) { @@ -647,3 +658,80 @@ int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; return TSDB_CODE_SUCCESS; } + +int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf) { + if (pLimitInfo->remainGroupOffset > 0) { + if (pLimitInfo->currentGroupId == 0) { // it is the first group + pLimitInfo->currentGroupId = pBlock->info.groupId; + blockDataCleanup(pBlock); + return PROJECT_RETRIEVE_CONTINUE; + } else if (pLimitInfo->currentGroupId != pBlock->info.groupId) { + // now it is the data from a new group + pLimitInfo->remainGroupOffset -= 1; + + // ignore data block in current group + if (pLimitInfo->remainGroupOffset > 0) { + blockDataCleanup(pBlock); + return PROJECT_RETRIEVE_CONTINUE; + } + } + + // set current group id of the project operator + pLimitInfo->currentGroupId = pBlock->info.groupId; + } + + // here check for a new group data, we need to handle the data of the previous group. + if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.groupId) { + pLimitInfo->numOfOutputGroups += 1; + if ((pLimitInfo->slimit.limit > 0) && (pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) { + pOperator->status = OP_EXEC_DONE; + blockDataCleanup(pBlock); + + return PROJECT_RETRIEVE_DONE; + } + + // reset the value for a new group data + pLimitInfo->numOfOutputRows = 0; + pLimitInfo->remainOffset = pLimitInfo->limit.offset; + + // existing rows that belongs to previous group. + if (pBlock->info.rows > 0) { + return PROJECT_RETRIEVE_DONE; + } + } + + // here we reach the start position, according to the limit/offset requirements. + + // set current group id + pLimitInfo->currentGroupId = pBlock->info.groupId; + + if (pLimitInfo->remainOffset >= pBlock->info.rows) { + pLimitInfo->remainOffset -= pBlock->info.rows; + blockDataCleanup(pBlock); + return PROJECT_RETRIEVE_CONTINUE; + } else if (pLimitInfo->remainOffset < pBlock->info.rows && pLimitInfo->remainOffset > 0) { + blockDataTrimFirstNRows(pBlock, pLimitInfo->remainOffset); + pLimitInfo->remainOffset = 0; + } + + // check for the limitation in each group + if (pLimitInfo->limit.limit >= 0 && pLimitInfo->numOfOutputRows + pBlock->info.rows >= pLimitInfo->limit.limit) { + int32_t keepRows = (int32_t)(pLimitInfo->limit.limit - pLimitInfo->numOfOutputRows); + blockDataKeepFirstNRows(pBlock, keepRows); + if (pLimitInfo->slimit.limit > 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups) { + pOperator->status = OP_EXEC_DONE; + } + + return PROJECT_RETRIEVE_DONE; + } + + // todo optimize performance + // If there are slimit/soffset value exists, multi-round result can not be packed into one group, since the + // they may not belong to the same group the limit/offset value is not valid in this case. + if ((!holdDataInBuf) || (pBlock->info.rows >= pOperator->resultInfo.threshold) || pLimitInfo->slimit.offset != -1 || + pLimitInfo->slimit.limit != -1) { + return PROJECT_RETRIEVE_DONE; + } else { // not full enough, continue to accumulate the output data in the buffer. + return PROJECT_RETRIEVE_CONTINUE; + } +} diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a7e955100c..9b1bc81dbe 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -24,7 +24,6 @@ #include "tdatablock.h" #include "tglobal.h" #include "tmsg.h" -#include "tsort.h" #include "ttime.h" #include "executorimpl.h" @@ -297,8 +296,6 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow colDataAppendInt64(pColData, 4, &pQueryWindow->ekey); } -void cleanupExecTimeWindowInfo(SColumnInfoData* pColData) { colDataDestroy(pColData); } - typedef struct { bool hasAgg; int32_t numOfRows; @@ -1347,42 +1344,6 @@ void queryCostStatis(SExecTaskInfo* pTaskInfo) { } } -// static void updateOffsetVal(STaskRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { -// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; -// STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; -// -// int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); -// -// if (pQueryAttr->limit.offset == pBlockInfo->rows) { // current block will ignore completed -// pTableQueryInfo->lastKey = QUERY_IS_ASC_QUERY(pQueryAttr) ? pBlockInfo->window.ekey + step : -// pBlockInfo->window.skey + step; pQueryAttr->limit.offset = 0; return; -// } -// -// if (QUERY_IS_ASC_QUERY(pQueryAttr)) { -// pQueryAttr->pos = (int32_t)pQueryAttr->limit.offset; -// } else { -// pQueryAttr->pos = pBlockInfo->rows - (int32_t)pQueryAttr->limit.offset - 1; -// } -// -// assert(pQueryAttr->pos >= 0 && pQueryAttr->pos <= pBlockInfo->rows - 1); -// -// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); -// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); -// -// // update the pQueryAttr->limit.offset value, and pQueryAttr->pos value -// TSKEY *keys = (TSKEY *) pColInfoData->pData; -// -// // update the offset value -// pTableQueryInfo->lastKey = keys[pQueryAttr->pos]; -// pQueryAttr->limit.offset = 0; -// -// int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); -// -// //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numBlocksOfStep:%d, numOfRes:%d, -// lastKey:%"PRId64, GET_TASKID(pRuntimeEnv), -// pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, pQuery->current->lastKey); -// } - // void skipBlocks(STaskRuntimeEnv *pRuntimeEnv) { // STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; // @@ -1723,159 +1684,6 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { return (rows == 0) ? NULL : pInfo->pRes; } -int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* length) { - if (result == NULL || length == NULL) { - return TSDB_CODE_TSC_INVALID_INPUT; - } - SOptrBasicInfo* pInfo = (SOptrBasicInfo*)(pOperator->info); - SAggSupporter* pSup = (SAggSupporter*)POINTER_SHIFT(pOperator->info, sizeof(SOptrBasicInfo)); - int32_t size = tSimpleHashGetSize(pSup->pResultRowHashTable); - size_t keyLen = sizeof(uint64_t) * 2; // estimate the key length - int32_t totalSize = - sizeof(int32_t) + sizeof(int32_t) + size * (sizeof(int32_t) + keyLen + sizeof(int32_t) + pSup->resultRowSize); - - // no result - if (getTotalBufSize(pSup->pResultBuf) == 0) { - *result = NULL; - *length = 0; - return TSDB_CODE_SUCCESS; - } - - *result = (char*)taosMemoryCalloc(1, totalSize); - if (*result == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - int32_t offset = sizeof(int32_t); - *(int32_t*)(*result + offset) = size; - offset += sizeof(int32_t); - - // prepare memory - SResultRowPosition* pos = &pInfo->resultRowInfo.cur; - void* pPage = getBufPage(pSup->pResultBuf, pos->pageId); - SResultRow* pRow = (SResultRow*)((char*)pPage + pos->offset); - setBufPageDirty(pPage, true); - releaseBufPage(pSup->pResultBuf, pPage); - - int32_t iter = 0; - void* pIter = NULL; - while ((pIter = tSimpleHashIterate(pSup->pResultRowHashTable, pIter, &iter))) { - void* key = tSimpleHashGetKey(pIter, &keyLen); - SResultRowPosition* p1 = (SResultRowPosition*)pIter; - - pPage = (SFilePage*)getBufPage(pSup->pResultBuf, p1->pageId); - pRow = (SResultRow*)((char*)pPage + p1->offset); - setBufPageDirty(pPage, true); - releaseBufPage(pSup->pResultBuf, pPage); - - // recalculate the result size - int32_t realTotalSize = offset + sizeof(int32_t) + keyLen + sizeof(int32_t) + pSup->resultRowSize; - if (realTotalSize > totalSize) { - char* tmp = (char*)taosMemoryRealloc(*result, realTotalSize); - if (tmp == NULL) { - taosMemoryFree(*result); - *result = NULL; - return TSDB_CODE_OUT_OF_MEMORY; - } else { - *result = tmp; - } - } - // save key - *(int32_t*)(*result + offset) = keyLen; - offset += sizeof(int32_t); - memcpy(*result + offset, key, keyLen); - offset += keyLen; - - // save value - *(int32_t*)(*result + offset) = pSup->resultRowSize; - offset += sizeof(int32_t); - memcpy(*result + offset, pRow, pSup->resultRowSize); - offset += pSup->resultRowSize; - } - - *(int32_t*)(*result) = offset; - *length = offset; - - return TDB_CODE_SUCCESS; -} - -int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf) { - if (pLimitInfo->remainGroupOffset > 0) { - if (pLimitInfo->currentGroupId == 0) { // it is the first group - pLimitInfo->currentGroupId = pBlock->info.groupId; - blockDataCleanup(pBlock); - return PROJECT_RETRIEVE_CONTINUE; - } else if (pLimitInfo->currentGroupId != pBlock->info.groupId) { - // now it is the data from a new group - pLimitInfo->remainGroupOffset -= 1; - - // ignore data block in current group - if (pLimitInfo->remainGroupOffset > 0) { - blockDataCleanup(pBlock); - return PROJECT_RETRIEVE_CONTINUE; - } - } - - // set current group id of the project operator - pLimitInfo->currentGroupId = pBlock->info.groupId; - } - - // here check for a new group data, we need to handle the data of the previous group. - if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.groupId) { - pLimitInfo->numOfOutputGroups += 1; - if ((pLimitInfo->slimit.limit > 0) && (pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) { - pOperator->status = OP_EXEC_DONE; - blockDataCleanup(pBlock); - - return PROJECT_RETRIEVE_DONE; - } - - // reset the value for a new group data - pLimitInfo->numOfOutputRows = 0; - pLimitInfo->remainOffset = pLimitInfo->limit.offset; - - // existing rows that belongs to previous group. - if (pBlock->info.rows > 0) { - return PROJECT_RETRIEVE_DONE; - } - } - - // here we reach the start position, according to the limit/offset requirements. - - // set current group id - pLimitInfo->currentGroupId = pBlock->info.groupId; - - if (pLimitInfo->remainOffset >= pBlock->info.rows) { - pLimitInfo->remainOffset -= pBlock->info.rows; - blockDataCleanup(pBlock); - return PROJECT_RETRIEVE_CONTINUE; - } else if (pLimitInfo->remainOffset < pBlock->info.rows && pLimitInfo->remainOffset > 0) { - blockDataTrimFirstNRows(pBlock, pLimitInfo->remainOffset); - pLimitInfo->remainOffset = 0; - } - - // check for the limitation in each group - if (pLimitInfo->limit.limit >= 0 && pLimitInfo->numOfOutputRows + pBlock->info.rows >= pLimitInfo->limit.limit) { - int32_t keepRows = (int32_t)(pLimitInfo->limit.limit - pLimitInfo->numOfOutputRows); - blockDataKeepFirstNRows(pBlock, keepRows); - if (pLimitInfo->slimit.limit > 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups) { - pOperator->status = OP_EXEC_DONE; - } - - return PROJECT_RETRIEVE_DONE; - } - - // todo optimize performance - // If there are slimit/soffset value exists, multi-round result can not be packed into one group, since the - // they may not belong to the same group the limit/offset value is not valid in this case. - if ((!holdDataInBuf) || (pBlock->info.rows >= pOperator->resultInfo.threshold) || pLimitInfo->slimit.offset != -1 || - pLimitInfo->slimit.limit != -1) { - return PROJECT_RETRIEVE_DONE; - } else { // not full enough, continue to accumulate the output data in the buffer. - return PROJECT_RETRIEVE_CONTINUE; - } -} - static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag); static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo, SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) { diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 26a5f6838d..fcfb79eb4b 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -27,6 +27,36 @@ #include "thash.h" #include "ttypes.h" +typedef struct SGroupbyOperatorInfo { + SOptrBasicInfo binfo; + SAggSupporter aggSup; + SArray* pGroupCols; // group by columns, SArray + SArray* pGroupColVals; // current group column values, SArray + bool isInit; // denote if current val is initialized or not + char* keyBuf; // group by keys for hash + int32_t groupKeyLen; // total group by column width + SGroupResInfo groupResInfo; + SExprSupp scalarSup; +} SGroupbyOperatorInfo; + +// The sort in partition may be needed later. +typedef struct SPartitionOperatorInfo { + SOptrBasicInfo binfo; + SArray* pGroupCols; + SArray* pGroupColVals; // current group column values, SArray + char* keyBuf; // group by keys for hash + int32_t groupKeyLen; // total group by column width + SHashObj* pGroupSet; // quick locate the window object for each result + + SDiskbasedBuf* pBuf; // query result buffer based on blocked-wised disk file + int32_t rowCapacity; // maximum number of rows for each buffer page + int32_t* columnOffset; // start position for each column data + SArray* sortedGroupArray; // SDataGroupInfo sorted by group id + int32_t groupIndex; // group index + int32_t pageIndex; // page index of current group + SExprSupp scalarSup; +} SPartitionOperatorInfo; + static void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInfo** pGroupInfo, int32_t len); static int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity); static int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 4e1daac643..61dc7a1b76 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -24,6 +24,21 @@ #include "tmsg.h" #include "ttypes.h" +typedef struct SJoinOperatorInfo { + SSDataBlock* pRes; + int32_t joinType; + int32_t inputOrder; + + SSDataBlock* pLeft; + int32_t leftPos; + SColumnInfo leftCol; + + SSDataBlock* pRight; + int32_t rightPos; + SColumnInfo rightCol; + SNode* pCondAfterMerge; +} SJoinOperatorInfo; + static void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode); static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator); static void destroyMergeJoinOperator(void* param); diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index ce1d13775c..ada7964c67 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -17,6 +17,24 @@ #include "executorimpl.h" #include "functionMgt.h" +typedef struct SProjectOperatorInfo { + SOptrBasicInfo binfo; + SAggSupporter aggSup; + SArray* pPseudoColInfo; + SLimitInfo limitInfo; + bool mergeDataBlocks; + SSDataBlock* pFinalRes; +} SProjectOperatorInfo; + +typedef struct SIndefOperatorInfo { + SOptrBasicInfo binfo; + SAggSupporter aggSup; + SArray* pPseudoColInfo; + SExprSupp scalarSup; + uint64_t groupId; + SSDataBlock* pNextGroupRes; +} SIndefOperatorInfo; + static SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator); static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator); static SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 916b6df969..24ca3e2eed 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include #include "executorimpl.h" #include "filter.h" #include "function.h" @@ -33,31 +32,17 @@ #include "ttypes.h" #include "vnode.h" +typedef struct SBlockDistInfo { + SSDataBlock* pResBlock; + STsdbReader* pHandle; + SReadHandle readHandle; + uint64_t uid; // table uid +} SBlockDistInfo; + #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) -static int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity); -static int32_t buildDbTableInfoBlock(bool sysInfo, const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, - size_t size, const char* dbName); - -static char* SYSTABLE_IDX_COLUMN[] = {"table_name", "db_name", "create_time", "columns", - "ttl", "stable_name", "vgroup_id', 'uid", "type"}; - -static char* SYSTABLE_SPECIAL_COL[] = {"db_name", "vgroup_id"}; - -typedef int32_t (*__sys_filte)(void* pMeta, SNode* cond, SArray* result); -typedef int32_t (*__sys_check)(SNode* cond); -typedef struct { - const char* name; - __sys_check chkFunc; - __sys_filte fltFunc; -} SSTabFltFuncDef; - -typedef struct { - void* pMeta; - void* pVnode; -} SSTabFltArg; typedef struct STableMergeScanExecInfo { SFileBlockLoadRecorder blockRecorder; @@ -71,54 +56,8 @@ typedef struct STableMergeScanSortSourceParam { SSDataBlock* inputBlock; } STableMergeScanSortSourceParam; -static int32_t sysChkFilter__Comm(SNode* pNode); -static int32_t sysChkFilter__DBName(SNode* pNode); -static int32_t sysChkFilter__VgroupId(SNode* pNode); -static int32_t sysChkFilter__TableName(SNode* pNode); -static int32_t sysChkFilter__CreateTime(SNode* pNode); -static int32_t sysChkFilter__Ncolumn(SNode* pNode); -static int32_t sysChkFilter__Ttl(SNode* pNode); -static int32_t sysChkFilter__STableName(SNode* pNode); -static int32_t sysChkFilter__Uid(SNode* pNode); -static int32_t sysChkFilter__Type(SNode* pNode); - -static int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result); -static int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result); -static int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result); -static int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result); -static int32_t sysFilte__Ncolumn(void* arg, SNode* pNode, SArray* result); -static int32_t sysFilte__Ttl(void* arg, SNode* pNode, SArray* result); -static int32_t sysFilte__STableName(void* arg, SNode* pNode, SArray* result); -static int32_t sysFilte__Uid(void* arg, SNode* pNode, SArray* result); -static int32_t sysFilte__Type(void* arg, SNode* pNode, SArray* result); - -const SSTabFltFuncDef filterDict[] = { - {.name = "table_name", .chkFunc = sysChkFilter__TableName, .fltFunc = sysFilte__TableName}, - {.name = "db_name", .chkFunc = sysChkFilter__DBName, .fltFunc = sysFilte__DbName}, - {.name = "create_time", .chkFunc = sysChkFilter__CreateTime, .fltFunc = sysFilte__CreateTime}, - {.name = "columns", .chkFunc = sysChkFilter__Ncolumn, .fltFunc = sysFilte__Ncolumn}, - {.name = "ttl", .chkFunc = sysChkFilter__Ttl, .fltFunc = sysFilte__Ttl}, - {.name = "stable_name", .chkFunc = sysChkFilter__STableName, .fltFunc = sysFilte__STableName}, - {.name = "vgroup_id", .chkFunc = sysChkFilter__VgroupId, .fltFunc = sysFilte__VgroupId}, - {.name = "uid", .chkFunc = sysChkFilter__Uid, .fltFunc = sysFilte__Uid}, - {.name = "type", .chkFunc = sysChkFilter__Type, .fltFunc = sysFilte__Type}}; - -#define SYSTAB_FILTER_DICT_SIZE (sizeof(filterDict) / sizeof(filterDict[0])) - -static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result); -static int32_t optSysTabFilteImpl(void* arg, SNode* cond, SArray* result); -static int32_t optSysCheckOper(SNode* pOpear); -static int32_t optSysMergeRslt(SArray* mRslt, SArray* rslt); - static bool processBlockWithProbability(const SSampleExecInfo* pInfo); -static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrSuperTable, - SMetaReader* smrChildTable, const char* dbname, const char* tableName, - int32_t* pNumOfRows, const SSDataBlock* dataBlock); - -static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t numOfRows, SSDataBlock* dataBlock, - SFilterInfo* pFilterInfo); - bool processBlockWithProbability(const SSampleExecInfo* pInfo) { #if 0 if (pInfo->sampleRatio == 1) { @@ -2615,1638 +2554,6 @@ _error: return NULL; } -static void destroySysScanOperator(void* param) { - SSysTableScanInfo* pInfo = (SSysTableScanInfo*)param; - tsem_destroy(&pInfo->ready); - blockDataDestroy(pInfo->pRes); - - const char* name = tNameGetTableName(&pInfo->name); - if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || - strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) { - metaCloseTbCursor(pInfo->pCur); - pInfo->pCur = NULL; - } - if (pInfo->pIdx) { - taosArrayDestroy(pInfo->pIdx->uids); - taosMemoryFree(pInfo->pIdx); - pInfo->pIdx = NULL; - } - - taosArrayDestroy(pInfo->matchInfo.pList); - taosMemoryFreeClear(pInfo->pUser); - - taosMemoryFreeClear(param); -} - -static int32_t getSysTableDbNameColId(const char* pTable) { - // if (0 == strcmp(TSDB_INS_TABLE_INDEXES, pTable)) { - // return 1; - // } - return TSDB_INS_USER_STABLES_DBNAME_COLID; -} - -EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) { - int32_t code = TSDB_CODE_SUCCESS; - ENodeType nType = nodeType(pNode); - - switch (nType) { - case QUERY_NODE_OPERATOR: { - SOperatorNode* node = (SOperatorNode*)pNode; - if (OP_TYPE_EQUAL == node->opType) { - *(int32_t*)pContext = 1; - return DEAL_RES_CONTINUE; - } - - *(int32_t*)pContext = 0; - return DEAL_RES_IGNORE_CHILD; - } - case QUERY_NODE_COLUMN: { - if (1 != *(int32_t*)pContext) { - return DEAL_RES_CONTINUE; - } - - SColumnNode* node = (SColumnNode*)pNode; - if (getSysTableDbNameColId(node->tableName) == node->colId) { - *(int32_t*)pContext = 2; - return DEAL_RES_CONTINUE; - } - - *(int32_t*)pContext = 0; - return DEAL_RES_CONTINUE; - } - case QUERY_NODE_VALUE: { - if (2 != *(int32_t*)pContext) { - return DEAL_RES_CONTINUE; - } - - SValueNode* node = (SValueNode*)pNode; - char* dbName = nodesGetValueFromNode(node); - strncpy(pContext, varDataVal(dbName), varDataLen(dbName)); - *((char*)pContext + varDataLen(dbName)) = 0; - return DEAL_RES_END; // stop walk - } - default: - break; - } - return DEAL_RES_CONTINUE; -} - -static void getDBNameFromCondition(SNode* pCondition, const char* dbName) { - if (NULL == pCondition) { - return; - } - nodesWalkExpr(pCondition, getDBNameFromConditionWalker, (char*)dbName); -} - -static int32_t loadSysTableCallback(void* param, SDataBuf* pMsg, int32_t code) { - SOperatorInfo* operator=(SOperatorInfo*) param; - SSysTableScanInfo* pScanResInfo = (SSysTableScanInfo*)operator->info; - if (TSDB_CODE_SUCCESS == code) { - pScanResInfo->pRsp = pMsg->pData; - - SRetrieveMetaTableRsp* pRsp = pScanResInfo->pRsp; - pRsp->numOfRows = htonl(pRsp->numOfRows); - pRsp->useconds = htobe64(pRsp->useconds); - pRsp->handle = htobe64(pRsp->handle); - pRsp->compLen = htonl(pRsp->compLen); - } else { - operator->pTaskInfo->code = code; - } - - tsem_post(&pScanResInfo->ready); - return TSDB_CODE_SUCCESS; -} - -static SSDataBlock* doFilterResult(SSDataBlock* pDataBlock, SFilterInfo* pFilterInfo) { - if (pFilterInfo == NULL) { - return pDataBlock->info.rows == 0 ? NULL : pDataBlock; - } - - doFilter(pDataBlock, pFilterInfo, NULL); - return pDataBlock->info.rows == 0 ? NULL : pDataBlock; -} - -static SSDataBlock* buildInfoSchemaTableMetaBlock(char* tableName) { - size_t size = 0; - const SSysTableMeta* pMeta = NULL; - getInfosDbMeta(&pMeta, &size); - - int32_t index = 0; - for (int32_t i = 0; i < size; ++i) { - if (strcmp(pMeta[i].name, tableName) == 0) { - index = i; - break; - } - } - - SSDataBlock* pBlock = createDataBlock(); - for (int32_t i = 0; i < pMeta[index].colNum; ++i) { - SColumnInfoData colInfoData = - createColumnInfoData(pMeta[index].schema[i].type, pMeta[index].schema[i].bytes, i + 1); - blockDataAppendColInfo(pBlock, &colInfoData); - } - - return pBlock; -} - -int32_t convertTagDataToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len) { - int32_t n = 0; - - switch (type) { - case TSDB_DATA_TYPE_NULL: - n = sprintf(str, "null"); - break; - - case TSDB_DATA_TYPE_BOOL: - n = sprintf(str, (*(int8_t*)buf) ? "true" : "false"); - break; - - case TSDB_DATA_TYPE_TINYINT: - n = sprintf(str, "%d", *(int8_t*)buf); - break; - - case TSDB_DATA_TYPE_SMALLINT: - n = sprintf(str, "%d", *(int16_t*)buf); - break; - - case TSDB_DATA_TYPE_INT: - n = sprintf(str, "%d", *(int32_t*)buf); - break; - - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: - n = sprintf(str, "%" PRId64, *(int64_t*)buf); - break; - - case TSDB_DATA_TYPE_FLOAT: - n = sprintf(str, "%.5f", GET_FLOAT_VAL(buf)); - break; - - case TSDB_DATA_TYPE_DOUBLE: - n = sprintf(str, "%.9f", GET_DOUBLE_VAL(buf)); - break; - - case TSDB_DATA_TYPE_BINARY: - if (bufSize < 0) { - return TSDB_CODE_TSC_INVALID_VALUE; - } - - memcpy(str, buf, bufSize); - n = bufSize; - break; - case TSDB_DATA_TYPE_NCHAR: - if (bufSize < 0) { - return TSDB_CODE_TSC_INVALID_VALUE; - } - - int32_t length = taosUcs4ToMbs((TdUcs4*)buf, bufSize, str); - if (length <= 0) { - return TSDB_CODE_TSC_INVALID_VALUE; - } - n = length; - break; - case TSDB_DATA_TYPE_UTINYINT: - n = sprintf(str, "%u", *(uint8_t*)buf); - break; - - case TSDB_DATA_TYPE_USMALLINT: - n = sprintf(str, "%u", *(uint16_t*)buf); - break; - - case TSDB_DATA_TYPE_UINT: - n = sprintf(str, "%u", *(uint32_t*)buf); - break; - - case TSDB_DATA_TYPE_UBIGINT: - n = sprintf(str, "%" PRIu64, *(uint64_t*)buf); - break; - - default: - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if (len) *len = n; - - return TSDB_CODE_SUCCESS; -} - -static bool sysTableIsOperatorCondOnOneTable(SNode* pCond, char* condTable) { - SOperatorNode* node = (SOperatorNode*)pCond; - if (node->opType == OP_TYPE_EQUAL) { - if (nodeType(node->pLeft) == QUERY_NODE_COLUMN && - strcasecmp(nodesGetNameFromColumnNode(node->pLeft), "table_name") == 0 && - nodeType(node->pRight) == QUERY_NODE_VALUE) { - SValueNode* pValue = (SValueNode*)node->pRight; - if (pValue->node.resType.type == TSDB_DATA_TYPE_NCHAR || pValue->node.resType.type == TSDB_DATA_TYPE_VARCHAR || - pValue->node.resType.type == TSDB_DATA_TYPE_BINARY) { - char* value = nodesGetValueFromNode(pValue); - strncpy(condTable, varDataVal(value), TSDB_TABLE_NAME_LEN); - return true; - } - } - } - return false; -} - -static bool sysTableIsCondOnOneTable(SNode* pCond, char* condTable) { - if (pCond == NULL) { - return false; - } - if (nodeType(pCond) == QUERY_NODE_LOGIC_CONDITION) { - SLogicConditionNode* node = (SLogicConditionNode*)pCond; - if (LOGIC_COND_TYPE_AND == node->condType) { - SNode* pChild = NULL; - FOREACH(pChild, node->pParameterList) { - if (QUERY_NODE_OPERATOR == nodeType(pChild) && sysTableIsOperatorCondOnOneTable(pChild, condTable)) { - return true; - } - } - } - } - - if (QUERY_NODE_OPERATOR == nodeType(pCond)) { - return sysTableIsOperatorCondOnOneTable(pCond, condTable); - } - - return false; -} - -static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SSysTableScanInfo* pInfo = pOperator->info; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - blockDataCleanup(pInfo->pRes); - int32_t numOfRows = 0; - - SSDataBlock* dataBlock = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TAGS); - blockDataEnsureCapacity(dataBlock, pOperator->resultInfo.capacity); - - const char* db = NULL; - int32_t vgId = 0; - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); - - SName sn = {0}; - char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); - - tNameGetDbName(&sn, varDataVal(dbname)); - varDataSetLen(dbname, strlen(varDataVal(dbname))); - - char condTableName[TSDB_TABLE_NAME_LEN] = {0}; - // optimize when sql like where table_name='tablename' and xxx. - if (pInfo->pCondition && sysTableIsCondOnOneTable(pInfo->pCondition, condTableName)) { - char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(tableName, condTableName); - - SMetaReader smrChildTable = {0}; - metaReaderInit(&smrChildTable, pInfo->readHandle.meta, 0); - int32_t code = metaGetTableEntryByName(&smrChildTable, condTableName); - if (code != TSDB_CODE_SUCCESS) { - // terrno has been set by metaGetTableEntryByName, therefore, return directly - return NULL; - } - - if (smrChildTable.me.type != TSDB_CHILD_TABLE) { - metaReaderClear(&smrChildTable); - blockDataDestroy(dataBlock); - pInfo->loadInfo.totalRows = 0; - return NULL; - } - - SMetaReader smrSuperTable = {0}; - metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, META_READER_NOLOCK); - code = metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); - if (code != TSDB_CODE_SUCCESS) { - // terrno has been set by metaGetTableEntryByUid - return NULL; - } - - sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock); - metaReaderClear(&smrSuperTable); - metaReaderClear(&smrChildTable); - if (numOfRows > 0) { - relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); - numOfRows = 0; - } - blockDataDestroy(dataBlock); - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - setOperatorCompleted(pOperator); - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; - } - - int32_t ret = 0; - if (pInfo->pCur == NULL) { - pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); - } - - while ((ret = metaTbCursorNext(pInfo->pCur)) == 0) { - if (pInfo->pCur->mr.me.type != TSDB_CHILD_TABLE) { - continue; - } - - char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); - - SMetaReader smrSuperTable = {0}; - metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0); - uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; - int32_t code = metaGetTableEntryByUid(&smrSuperTable, suid); - if (code != TSDB_CODE_SUCCESS) { - qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), - GET_TASKID(pTaskInfo)); - metaReaderClear(&smrSuperTable); - metaCloseTbCursor(pInfo->pCur); - pInfo->pCur = NULL; - T_LONG_JMP(pTaskInfo->env, terrno); - } - - sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows, dataBlock); - - metaReaderClear(&smrSuperTable); - - if (numOfRows >= pOperator->resultInfo.capacity) { - relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); - numOfRows = 0; - - if (pInfo->pRes->info.rows > 0) { - break; - } - } - } - - if (numOfRows > 0) { - relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); - numOfRows = 0; - } - - blockDataDestroy(dataBlock); - if (ret != 0) { - metaCloseTbCursor(pInfo->pCur); - pInfo->pCur = NULL; - setOperatorCompleted(pOperator); - } - - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; -} - -void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t numOfRows, SSDataBlock* dataBlock, - SFilterInfo* pFilterInfo) { - dataBlock->info.rows = numOfRows; - pInfo->pRes->info.rows = numOfRows; - - relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, dataBlock->pDataBlock, false); - doFilterResult(pInfo->pRes, pFilterInfo); - blockDataCleanup(dataBlock); -} - -static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrSuperTable, - SMetaReader* smrChildTable, const char* dbname, const char* tableName, - int32_t* pNumOfRows, const SSDataBlock* dataBlock) { - char stableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(stableName, (*smrSuperTable).me.name); - - int32_t numOfRows = *pNumOfRows; - - int32_t numOfTags = (*smrSuperTable).me.stbEntry.schemaTag.nCols; - for (int32_t i = 0; i < numOfTags; ++i) { - SColumnInfoData* pColInfoData = NULL; - - // table name - pColInfoData = taosArrayGet(dataBlock->pDataBlock, 0); - colDataAppend(pColInfoData, numOfRows, tableName, false); - - // database name - pColInfoData = taosArrayGet(dataBlock->pDataBlock, 1); - colDataAppend(pColInfoData, numOfRows, dbname, false); - - // super table name - pColInfoData = taosArrayGet(dataBlock->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, stableName, false); - - // tag name - char tagName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(tagName, (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].name); - pColInfoData = taosArrayGet(dataBlock->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, tagName, false); - - // tag type - int8_t tagType = (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].type; - pColInfoData = taosArrayGet(dataBlock->pDataBlock, 4); - char tagTypeStr[VARSTR_HEADER_SIZE + 32]; - int tagTypeLen = sprintf(varDataVal(tagTypeStr), "%s", tDataTypes[tagType].name); - if (tagType == TSDB_DATA_TYPE_VARCHAR) { - tagTypeLen += sprintf(varDataVal(tagTypeStr) + tagTypeLen, "(%d)", - (int32_t)((*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE)); - } else if (tagType == TSDB_DATA_TYPE_NCHAR) { - tagTypeLen += sprintf( - varDataVal(tagTypeStr) + tagTypeLen, "(%d)", - (int32_t)(((*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); - } - varDataSetLen(tagTypeStr, tagTypeLen); - colDataAppend(pColInfoData, numOfRows, (char*)tagTypeStr, false); - - STagVal tagVal = {0}; - tagVal.cid = (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].colId; - char* tagData = NULL; - uint32_t tagLen = 0; - - if (tagType == TSDB_DATA_TYPE_JSON) { - tagData = (char*)smrChildTable->me.ctbEntry.pTags; - } else { - bool exist = tTagGet((STag*)smrChildTable->me.ctbEntry.pTags, &tagVal); - if (exist) { - if (IS_VAR_DATA_TYPE(tagType)) { - tagData = (char*)tagVal.pData; - tagLen = tagVal.nData; - } else { - tagData = (char*)&tagVal.i64; - tagLen = tDataTypes[tagType].bytes; - } - } - } - - char* tagVarChar = NULL; - if (tagData != NULL) { - if (tagType == TSDB_DATA_TYPE_JSON) { - char* tagJson = parseTagDatatoJson(tagData); - tagVarChar = taosMemoryMalloc(strlen(tagJson) + VARSTR_HEADER_SIZE); - memcpy(varDataVal(tagVarChar), tagJson, strlen(tagJson)); - varDataSetLen(tagVarChar, strlen(tagJson)); - taosMemoryFree(tagJson); - } else { - int32_t bufSize = IS_VAR_DATA_TYPE(tagType) ? (tagLen + VARSTR_HEADER_SIZE) - : (3 + DBL_MANT_DIG - DBL_MIN_EXP + VARSTR_HEADER_SIZE); - tagVarChar = taosMemoryMalloc(bufSize); - int32_t len = -1; - convertTagDataToStr(varDataVal(tagVarChar), tagType, tagData, tagLen, &len); - varDataSetLen(tagVarChar, len); - } - } - pColInfoData = taosArrayGet(dataBlock->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, tagVarChar, - (tagData == NULL) || (tagType == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(tagData))); - taosMemoryFree(tagVarChar); - ++numOfRows; - } - - *pNumOfRows = numOfRows; - - return TSDB_CODE_SUCCESS; -} - -typedef int (*__optSysFilter)(void* a, void* b, int16_t dtype); - -int optSysDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) { - int32_t cmp = func(a, b); - switch (comparType) { - case OP_TYPE_LOWER_THAN: - if (cmp < 0) return 0; - break; - case OP_TYPE_LOWER_EQUAL: { - if (cmp <= 0) return 0; - break; - } - case OP_TYPE_GREATER_THAN: { - if (cmp > 0) return 0; - break; - } - case OP_TYPE_GREATER_EQUAL: { - if (cmp >= 0) return 0; - break; - } - case OP_TYPE_EQUAL: { - if (cmp == 0) return 0; - break; - } - default: - return -1; - } - return cmp; -} - -static int optSysFilterFuncImpl__LowerThan(void* a, void* b, int16_t dtype) { - __compar_fn_t func = getComparFunc(dtype, 0); - return optSysDoCompare(func, OP_TYPE_LOWER_THAN, a, b); -} -static int optSysFilterFuncImpl__LowerEqual(void* a, void* b, int16_t dtype) { - __compar_fn_t func = getComparFunc(dtype, 0); - return optSysDoCompare(func, OP_TYPE_LOWER_EQUAL, a, b); -} -static int optSysFilterFuncImpl__GreaterThan(void* a, void* b, int16_t dtype) { - __compar_fn_t func = getComparFunc(dtype, 0); - return optSysDoCompare(func, OP_TYPE_GREATER_THAN, a, b); -} -static int optSysFilterFuncImpl__GreaterEqual(void* a, void* b, int16_t dtype) { - __compar_fn_t func = getComparFunc(dtype, 0); - return optSysDoCompare(func, OP_TYPE_GREATER_EQUAL, a, b); -} -static int optSysFilterFuncImpl__Equal(void* a, void* b, int16_t dtype) { - __compar_fn_t func = getComparFunc(dtype, 0); - return optSysDoCompare(func, OP_TYPE_EQUAL, a, b); -} - -static int optSysFilterFuncImpl__NoEqual(void* a, void* b, int16_t dtype) { - __compar_fn_t func = getComparFunc(dtype, 0); - return optSysDoCompare(func, OP_TYPE_NOT_EQUAL, a, b); -} -static __optSysFilter optSysGetFilterFunc(int32_t ctype, bool* reverse) { - if (ctype == OP_TYPE_LOWER_EQUAL || ctype == OP_TYPE_LOWER_THAN) { - *reverse = true; - } - if (ctype == OP_TYPE_LOWER_THAN) - return optSysFilterFuncImpl__LowerThan; - else if (ctype == OP_TYPE_LOWER_EQUAL) - return optSysFilterFuncImpl__LowerEqual; - else if (ctype == OP_TYPE_GREATER_THAN) - return optSysFilterFuncImpl__GreaterThan; - else if (ctype == OP_TYPE_GREATER_EQUAL) - return optSysFilterFuncImpl__GreaterEqual; - else if (ctype == OP_TYPE_EQUAL) - return optSysFilterFuncImpl__Equal; - else if (ctype == OP_TYPE_NOT_EQUAL) - return optSysFilterFuncImpl__NoEqual; - return NULL; -} -static int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result) { - void* pVnode = ((SSTabFltArg*)arg)->pVnode; - - const char* db = NULL; - vnodeGetInfo(pVnode, &db, NULL); - - SName sn = {0}; - char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); - - tNameGetDbName(&sn, varDataVal(dbname)); - varDataSetLen(dbname, strlen(varDataVal(dbname))); - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - - bool reverse = false; - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - - int ret = func(dbname, pVal->datum.p, TSDB_DATA_TYPE_VARCHAR); - if (ret == 0) return 0; - - return -2; -} -static int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result) { - void* pVnode = ((SSTabFltArg*)arg)->pVnode; - - int64_t vgId = 0; - vnodeGetInfo(pVnode, NULL, (int32_t*)&vgId); - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - - bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - - int ret = func(&vgId, &pVal->datum.i, TSDB_DATA_TYPE_BIGINT); - if (ret == 0) return 0; - - return -1; -} -static int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - - SMetaFltParam param = {.suid = 0, - .cid = 0, - .type = TSDB_DATA_TYPE_VARCHAR, - .val = pVal->datum.p, - .reverse = reverse, - .filterFunc = func}; - return -1; -} - -static int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - - SMetaFltParam param = {.suid = 0, - .cid = 0, - .type = TSDB_DATA_TYPE_BIGINT, - .val = &pVal->datum.i, - .reverse = reverse, - .filterFunc = func}; - - int32_t ret = metaFilterCreateTime(pMeta, ¶m, result); - return ret; -} -static int32_t sysFilte__Ncolumn(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - return -1; -} - -static int32_t sysFilte__Ttl(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - return -1; -} -static int32_t sysFilte__STableName(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - return -1; -} -static int32_t sysFilte__Uid(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - return -1; -} -static int32_t sysFilte__Type(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - return -1; -} -static int32_t sysChkFilter__Comm(SNode* pNode) { - // impl - SOperatorNode* pOper = (SOperatorNode*)pNode; - EOperatorType opType = pOper->opType; - if (opType != OP_TYPE_EQUAL && opType != OP_TYPE_LOWER_EQUAL && opType != OP_TYPE_LOWER_THAN && - opType != OP_TYPE_GREATER_EQUAL && opType != OP_TYPE_GREATER_THAN) { - return -1; - } - return 0; -} - -static int32_t sysChkFilter__DBName(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - - if (pOper->opType != OP_TYPE_EQUAL && pOper->opType != OP_TYPE_NOT_EQUAL) { - return -1; - } - - SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_STR_DATA_TYPE(pVal->node.resType.type)) { - return -1; - } - - return 0; -} -static int32_t sysChkFilter__VgroupId(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { - return -1; - } - return sysChkFilter__Comm(pNode); -} -static int32_t sysChkFilter__TableName(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_STR_DATA_TYPE(pVal->node.resType.type)) { - return -1; - } - return sysChkFilter__Comm(pNode); -} -static int32_t sysChkFilter__CreateTime(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - - if (!IS_TIMESTAMP_TYPE(pVal->node.resType.type)) { - return -1; - } - return sysChkFilter__Comm(pNode); -} - -static int32_t sysChkFilter__Ncolumn(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - - if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { - return -1; - } - return sysChkFilter__Comm(pNode); -} -static int32_t sysChkFilter__Ttl(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - - if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { - return -1; - } - return sysChkFilter__Comm(pNode); -} -static int32_t sysChkFilter__STableName(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_STR_DATA_TYPE(pVal->node.resType.type)) { - return -1; - } - return sysChkFilter__Comm(pNode); -} -static int32_t sysChkFilter__Uid(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { - return -1; - } - return sysChkFilter__Comm(pNode); -} -static int32_t sysChkFilter__Type(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { - return -1; - } - return sysChkFilter__Comm(pNode); -} -static int32_t optSysTabFilteImpl(void* arg, SNode* cond, SArray* result) { - if (optSysCheckOper(cond) != 0) return -1; - - SOperatorNode* pNode = (SOperatorNode*)cond; - - int8_t i = 0; - for (; i < SYSTAB_FILTER_DICT_SIZE; i++) { - if (strcmp(filterDict[i].name, ((SColumnNode*)(pNode->pLeft))->colName) == 0) { - break; - } - } - if (i >= SYSTAB_FILTER_DICT_SIZE) return -1; - - if (filterDict[i].chkFunc(cond) != 0) return -1; - - return filterDict[i].fltFunc(arg, cond, result); -} - -static int32_t optSysCheckOper(SNode* pOpear) { - if (nodeType(pOpear) != QUERY_NODE_OPERATOR) return -1; - - SOperatorNode* pOper = (SOperatorNode*)pOpear; - if (pOper->opType < OP_TYPE_GREATER_THAN || pOper->opType > OP_TYPE_NOT_EQUAL) { - return -1; - } - - if (nodeType(pOper->pLeft) != QUERY_NODE_COLUMN || nodeType(pOper->pRight) != QUERY_NODE_VALUE) { - return -1; - } - return 0; -} - -static int tableUidCompare(const void* a, const void* b) { - int64_t u1 = *(int64_t*)a; - int64_t u2 = *(int64_t*)b; - if (u1 == u2) { - return 0; - } - return u1 < u2 ? -1 : 1; -} - -typedef struct MergeIndex { - int idx; - int len; -} MergeIndex; - -static FORCE_INLINE int optSysBinarySearch(SArray* arr, int s, int e, uint64_t k) { - uint64_t v; - int32_t m; - while (s <= e) { - m = s + (e - s) / 2; - v = *(uint64_t*)taosArrayGet(arr, m); - if (v >= k) { - e = m - 1; - } else { - s = m + 1; - } - } - return s; -} - -void optSysIntersection(SArray* in, SArray* out) { - int32_t sz = (int32_t)taosArrayGetSize(in); - if (sz <= 0) { - return; - } - MergeIndex* mi = taosMemoryCalloc(sz, sizeof(MergeIndex)); - for (int i = 0; i < sz; i++) { - SArray* t = taosArrayGetP(in, i); - mi[i].len = (int32_t)taosArrayGetSize(t); - mi[i].idx = 0; - } - - SArray* base = taosArrayGetP(in, 0); - for (int i = 0; i < taosArrayGetSize(base); i++) { - uint64_t tgt = *(uint64_t*)taosArrayGet(base, i); - bool has = true; - for (int j = 1; j < taosArrayGetSize(in); j++) { - SArray* oth = taosArrayGetP(in, j); - int mid = optSysBinarySearch(oth, mi[j].idx, mi[j].len - 1, tgt); - if (mid >= 0 && mid < mi[j].len) { - uint64_t val = *(uint64_t*)taosArrayGet(oth, mid); - has = (val == tgt ? true : false); - mi[j].idx = mid; - } else { - has = false; - } - } - if (has == true) { - taosArrayPush(out, &tgt); - } - } - taosMemoryFreeClear(mi); -} - -static int32_t optSysMergeRslt(SArray* mRslt, SArray* rslt) { - // TODO, find comm mem from mRslt - for (int i = 0; i < taosArrayGetSize(mRslt); i++) { - SArray* arslt = taosArrayGetP(mRslt, i); - taosArraySort(arslt, tableUidCompare); - } - optSysIntersection(mRslt, rslt); - return 0; -} - -static int32_t optSysSpecialColumn(SNode* cond) { - SOperatorNode* pOper = (SOperatorNode*)cond; - SColumnNode* pCol = (SColumnNode*)pOper->pLeft; - for (int i = 0; i < sizeof(SYSTABLE_SPECIAL_COL) / sizeof(SYSTABLE_SPECIAL_COL[0]); i++) { - if (0 == strcmp(pCol->colName, SYSTABLE_SPECIAL_COL[i])) { - return 1; - } - } - return 0; -} - -static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result) { - int ret = -1; - if (nodeType(cond) == QUERY_NODE_OPERATOR) { - ret = optSysTabFilteImpl(arg, cond, result); - if (ret == 0) { - SOperatorNode* pOper = (SOperatorNode*)cond; - SColumnNode* pCol = (SColumnNode*)pOper->pLeft; - if (0 == strcmp(pCol->colName, "create_time")) { - return 0; - } - return -1; - } - return ret; - } - - if (nodeType(cond) != QUERY_NODE_LOGIC_CONDITION || ((SLogicConditionNode*)cond)->condType != LOGIC_COND_TYPE_AND) { - return ret; - } - - SLogicConditionNode* pNode = (SLogicConditionNode*)cond; - SNodeList* pList = (SNodeList*)pNode->pParameterList; - - int32_t len = LIST_LENGTH(pList); - - bool hasIdx = false; - bool hasRslt = true; - SArray* mRslt = taosArrayInit(len, POINTER_BYTES); - - SListCell* cell = pList->pHead; - for (int i = 0; i < len; i++) { - if (cell == NULL) break; - - SArray* aRslt = taosArrayInit(16, sizeof(int64_t)); - - ret = optSysTabFilteImpl(arg, cell->pNode, aRslt); - if (ret == 0) { - // has index - hasIdx = true; - if (optSysSpecialColumn(cell->pNode) == 0) { - taosArrayPush(mRslt, &aRslt); - } else { - // db_name/vgroup not result - taosArrayDestroy(aRslt); - } - } else if (ret == -2) { - // current vg - hasIdx = true; - hasRslt = false; - taosArrayDestroy(aRslt); - break; - } else { - taosArrayDestroy(aRslt); - } - cell = cell->pNext; - } - if (hasRslt && hasIdx) { - optSysMergeRslt(mRslt, result); - } - - for (int i = 0; i < taosArrayGetSize(mRslt); i++) { - SArray* aRslt = taosArrayGetP(mRslt, i); - taosArrayDestroy(aRslt); - } - taosArrayDestroy(mRslt); - if (hasRslt == false) { - return -2; - } - if (hasRslt && hasIdx) { - cell = pList->pHead; - for (int i = 0; i < len; i++) { - if (cell == NULL) break; - SOperatorNode* pOper = (SOperatorNode*)cell->pNode; - SColumnNode* pCol = (SColumnNode*)pOper->pLeft; - if (0 == strcmp(pCol->colName, "create_time")) { - return 0; - } - cell = cell->pNext; - } - return -1; - } - return -1; -} - -static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SSysTableScanInfo* pInfo = pOperator->info; - - SSysTableIndex* pIdx = pInfo->pIdx; - blockDataCleanup(pInfo->pRes); - int32_t numOfRows = 0; - - int ret = 0; - - const char* db = NULL; - int32_t vgId = 0; - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); - - SName sn = {0}; - char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); - - tNameGetDbName(&sn, varDataVal(dbname)); - varDataSetLen(dbname, strlen(varDataVal(dbname))); - - SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TABLES); - blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); - - char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - int32_t i = pIdx->lastIdx; - for (; i < taosArrayGetSize(pIdx->uids); i++) { - tb_uid_t* uid = taosArrayGet(pIdx->uids, i); - - SMetaReader mr = {0}; - metaReaderInit(&mr, pInfo->readHandle.meta, 0); - ret = metaGetTableEntryByUid(&mr, *uid); - if (ret < 0) { - metaReaderClear(&mr); - continue; - } - STR_TO_VARSTR(n, mr.me.name); - - // table name - SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); - colDataAppend(pColInfoData, numOfRows, n, false); - - // database name - pColInfoData = taosArrayGet(p->pDataBlock, 1); - colDataAppend(pColInfoData, numOfRows, dbname, false); - - // vgId - pColInfoData = taosArrayGet(p->pDataBlock, 6); - colDataAppend(pColInfoData, numOfRows, (char*)&vgId, false); - - int32_t tableType = mr.me.type; - if (tableType == TSDB_CHILD_TABLE) { - // create time - int64_t ts = mr.me.ctbEntry.ctime; - pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, (char*)&ts, false); - - SMetaReader mr1 = {0}; - metaReaderInit(&mr1, pInfo->readHandle.meta, META_READER_NOLOCK); - - int64_t suid = mr.me.ctbEntry.suid; - int32_t code = metaGetTableEntryByUid(&mr1, suid); - if (code != TSDB_CODE_SUCCESS) { - qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name, - suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); - metaReaderClear(&mr1); - metaReaderClear(&mr); - T_LONG_JMP(pTaskInfo->env, terrno); - } - pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*)&mr1.me.stbEntry.schemaRow.nCols, false); - - // super table name - STR_TO_VARSTR(n, mr1.me.name); - pColInfoData = taosArrayGet(p->pDataBlock, 4); - colDataAppend(pColInfoData, numOfRows, n, false); - metaReaderClear(&mr1); - - // table comment - pColInfoData = taosArrayGet(p->pDataBlock, 8); - if (mr.me.ctbEntry.commentLen > 0) { - char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, mr.me.ctbEntry.comment); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else if (mr.me.ctbEntry.commentLen == 0) { - char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, ""); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else { - colDataAppendNULL(pColInfoData, numOfRows); - } - - // uid - pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.uid, false); - - // ttl - pColInfoData = taosArrayGet(p->pDataBlock, 7); - colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.ctbEntry.ttlDays, false); - - STR_TO_VARSTR(n, "CHILD_TABLE"); - - } else if (tableType == TSDB_NORMAL_TABLE) { - // create time - pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false); - - // number of columns - pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.schemaRow.nCols, false); - - // super table name - pColInfoData = taosArrayGet(p->pDataBlock, 4); - colDataAppendNULL(pColInfoData, numOfRows); - - // table comment - pColInfoData = taosArrayGet(p->pDataBlock, 8); - if (mr.me.ntbEntry.commentLen > 0) { - char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, mr.me.ntbEntry.comment); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else if (mr.me.ntbEntry.commentLen == 0) { - char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, ""); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else { - colDataAppendNULL(pColInfoData, numOfRows); - } - - // uid - pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.uid, false); - - // ttl - pColInfoData = taosArrayGet(p->pDataBlock, 7); - colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.ntbEntry.ttlDays, false); - - STR_TO_VARSTR(n, "NORMAL_TABLE"); - // impl later - } - - metaReaderClear(&mr); - - pColInfoData = taosArrayGet(p->pDataBlock, 9); - colDataAppend(pColInfoData, numOfRows, n, false); - - if (++numOfRows >= pOperator->resultInfo.capacity) { - p->info.rows = numOfRows; - pInfo->pRes->info.rows = numOfRows; - - relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); - doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); - - blockDataCleanup(p); - numOfRows = 0; - - if (pInfo->pRes->info.rows > 0) { - break; - } - } - } - - if (numOfRows > 0) { - p->info.rows = numOfRows; - pInfo->pRes->info.rows = numOfRows; - - relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); - doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); - - blockDataCleanup(p); - numOfRows = 0; - } - - if (i >= taosArrayGetSize(pIdx->uids)) { - setOperatorCompleted(pOperator); - } else { - pIdx->lastIdx = i + 1; - } - - blockDataDestroy(p); - - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; -} - -static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - SSysTableScanInfo* pInfo = pOperator->info; - if (pInfo->pCur == NULL) { - pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); - } - - blockDataCleanup(pInfo->pRes); - int32_t numOfRows = 0; - - const char* db = NULL; - int32_t vgId = 0; - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); - - SName sn = {0}; - char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); - - tNameGetDbName(&sn, varDataVal(dbname)); - varDataSetLen(dbname, strlen(varDataVal(dbname))); - - SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TABLES); - blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); - - char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - - int32_t ret = 0; - while ((ret = metaTbCursorNext(pInfo->pCur)) == 0) { - STR_TO_VARSTR(n, pInfo->pCur->mr.me.name); - - // table name - SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); - colDataAppend(pColInfoData, numOfRows, n, false); - - // database name - pColInfoData = taosArrayGet(p->pDataBlock, 1); - colDataAppend(pColInfoData, numOfRows, dbname, false); - - // vgId - pColInfoData = taosArrayGet(p->pDataBlock, 6); - colDataAppend(pColInfoData, numOfRows, (char*)&vgId, false); - - int32_t tableType = pInfo->pCur->mr.me.type; - if (tableType == TSDB_CHILD_TABLE) { - // create time - int64_t ts = pInfo->pCur->mr.me.ctbEntry.ctime; - pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, (char*)&ts, false); - - SMetaReader mr = {0}; - metaReaderInit(&mr, pInfo->readHandle.meta, META_READER_NOLOCK); - - uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; - int32_t code = metaGetTableEntryByUid(&mr, suid); - if (code != TSDB_CODE_SUCCESS) { - qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name, - suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); - metaReaderClear(&mr); - metaCloseTbCursor(pInfo->pCur); - pInfo->pCur = NULL; - T_LONG_JMP(pTaskInfo->env, terrno); - } - - // number of columns - pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.stbEntry.schemaRow.nCols, false); - - // super table name - STR_TO_VARSTR(n, mr.me.name); - pColInfoData = taosArrayGet(p->pDataBlock, 4); - colDataAppend(pColInfoData, numOfRows, n, false); - metaReaderClear(&mr); - - // table comment - pColInfoData = taosArrayGet(p->pDataBlock, 8); - if (pInfo->pCur->mr.me.ctbEntry.commentLen > 0) { - char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ctbEntry.comment); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else if (pInfo->pCur->mr.me.ctbEntry.commentLen == 0) { - char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, ""); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else { - colDataAppendNULL(pColInfoData, numOfRows); - } - - // uid - pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); - - // ttl - pColInfoData = taosArrayGet(p->pDataBlock, 7); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ctbEntry.ttlDays, false); - - STR_TO_VARSTR(n, "CHILD_TABLE"); - } else if (tableType == TSDB_NORMAL_TABLE) { - // create time - pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false); - - // number of columns - pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.schemaRow.nCols, false); - - // super table name - pColInfoData = taosArrayGet(p->pDataBlock, 4); - colDataAppendNULL(pColInfoData, numOfRows); - - // table comment - pColInfoData = taosArrayGet(p->pDataBlock, 8); - if (pInfo->pCur->mr.me.ntbEntry.commentLen > 0) { - char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ntbEntry.comment); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else if (pInfo->pCur->mr.me.ntbEntry.commentLen == 0) { - char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, ""); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else { - colDataAppendNULL(pColInfoData, numOfRows); - } - - // uid - pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); - - // ttl - pColInfoData = taosArrayGet(p->pDataBlock, 7); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ttlDays, false); - - STR_TO_VARSTR(n, "NORMAL_TABLE"); - } - - pColInfoData = taosArrayGet(p->pDataBlock, 9); - colDataAppend(pColInfoData, numOfRows, n, false); - - if (++numOfRows >= pOperator->resultInfo.capacity) { - p->info.rows = numOfRows; - pInfo->pRes->info.rows = numOfRows; - - relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); - doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); - - blockDataCleanup(p); - numOfRows = 0; - - if (pInfo->pRes->info.rows > 0) { - break; - } - } - } - - if (numOfRows > 0) { - p->info.rows = numOfRows; - pInfo->pRes->info.rows = numOfRows; - - relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); - doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); - - blockDataCleanup(p); - numOfRows = 0; - } - - blockDataDestroy(p); - - // todo temporarily free the cursor here, the true reason why the free is not valid needs to be found - if (ret != 0) { - metaCloseTbCursor(pInfo->pCur); - pInfo->pCur = NULL; - setOperatorCompleted(pOperator); - } - - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; -} - -static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SSysTableScanInfo* pInfo = pOperator->info; - - SNode* pCondition = pInfo->pCondition; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - // the retrieve is executed on the mnode, so return tables that belongs to the information schema database. - if (pInfo->readHandle.mnd != NULL) { - buildSysDbTableInfo(pInfo, pOperator->resultInfo.capacity); - doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - - setOperatorCompleted(pOperator); - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; - } else { - if (pInfo->showRewrite == false) { - if (pCondition != NULL && pInfo->pIdx == NULL) { - SSTabFltArg arg = {.pMeta = pInfo->readHandle.meta, .pVnode = pInfo->readHandle.vnode}; - - SSysTableIndex* idx = taosMemoryMalloc(sizeof(SSysTableIndex)); - idx->init = 0; - idx->uids = taosArrayInit(128, sizeof(int64_t)); - idx->lastIdx = 0; - - pInfo->pIdx = idx; // set idx arg - - int flt = optSysTabFilte(&arg, pCondition, idx->uids); - if (flt == 0) { - pInfo->pIdx->init = 1; - SSDataBlock* blk = sysTableBuildUserTablesByUids(pOperator); - return blk; - } else if (flt == -2) { - qDebug("%s failed to get sys table info by idx, empty result", GET_TASKID(pTaskInfo)); - return NULL; - } else if (flt == -1) { - // not idx - qDebug("%s failed to get sys table info by idx, scan sys table one by one", GET_TASKID(pTaskInfo)); - } - } else if (pCondition != NULL && (pInfo->pIdx != NULL && pInfo->pIdx->init == 1)) { - SSDataBlock* blk = sysTableBuildUserTablesByUids(pOperator); - return blk; - } - } - - return sysTableBuildUserTables(pOperator); - } - return NULL; -} - -static SSDataBlock* sysTableScanUserSTables(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SSysTableScanInfo* pInfo = pOperator->info; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - pInfo->pRes->info.rows = 0; - pOperator->status = OP_EXEC_DONE; - - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; -} - -static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { - // build message and send to mnode to fetch the content of system tables. - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SSysTableScanInfo* pInfo = pOperator->info; - char dbName[TSDB_DB_NAME_LEN] = {0}; - - const char* name = tNameGetTableName(&pInfo->name); - if (pInfo->showRewrite) { - getDBNameFromCondition(pInfo->pCondition, dbName); - sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName); - } - - if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { - return sysTableScanUserTables(pOperator); - } else if (strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { - return sysTableScanUserTags(pOperator); - } else if (strncasecmp(name, TSDB_INS_TABLE_STABLES, TSDB_TABLE_FNAME_LEN) == 0 && pInfo->showRewrite && - IS_SYS_DBNAME(dbName)) { - return sysTableScanUserSTables(pOperator); - } else { // load the meta from mnode of the given epset - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - while (1) { - int64_t startTs = taosGetTimestampUs(); - tstrncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); - tstrncpy(pInfo->req.user, pInfo->pUser, tListLen(pInfo->req.user)); - - int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req); - char* buf1 = taosMemoryCalloc(1, contLen); - tSerializeSRetrieveTableReq(buf1, contLen, &pInfo->req); - - // send the fetch remote task result reques - SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); - if (NULL == pMsgSendInfo) { - qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; - } - - int32_t msgType = (strcasecmp(name, TSDB_INS_TABLE_DNODE_VARIABLES) == 0) ? TDMT_DND_SYSTABLE_RETRIEVE - : TDMT_MND_SYSTABLE_RETRIEVE; - - pMsgSendInfo->param = pOperator; - pMsgSendInfo->msgInfo.pData = buf1; - pMsgSendInfo->msgInfo.len = contLen; - pMsgSendInfo->msgType = msgType; - pMsgSendInfo->fp = loadSysTableCallback; - pMsgSendInfo->requestId = pTaskInfo->id.queryId; - - int64_t transporterId = 0; - int32_t code = - asyncSendMsgToServer(pInfo->readHandle.pMsgCb->clientRpc, &pInfo->epSet, &transporterId, pMsgSendInfo); - tsem_wait(&pInfo->ready); - - if (pTaskInfo->code) { - qDebug("%s load meta data from mnode failed, totalRows:%" PRIu64 ", code:%s", GET_TASKID(pTaskInfo), - pInfo->loadInfo.totalRows, tstrerror(pTaskInfo->code)); - return NULL; - } - - SRetrieveMetaTableRsp* pRsp = pInfo->pRsp; - pInfo->req.showId = pRsp->handle; - - if (pRsp->numOfRows == 0 || pRsp->completed) { - pOperator->status = OP_EXEC_DONE; - qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64, GET_TASKID(pTaskInfo), - pRsp->numOfRows, pInfo->loadInfo.totalRows); - - if (pRsp->numOfRows == 0) { - taosMemoryFree(pRsp); - return NULL; - } - } - - char* pStart = pRsp->data; - extractDataBlockFromFetchRsp(pInfo->pRes, pRsp->data, pInfo->matchInfo.pList, &pStart); - updateLoadRemoteInfo(&pInfo->loadInfo, pRsp->numOfRows, pRsp->compLen, startTs, pOperator); - - // todo log the filter info - doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); - taosMemoryFree(pRsp); - if (pInfo->pRes->info.rows > 0) { - return pInfo->pRes; - } else if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - } - } -} - -int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity) { - SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TABLES); - blockDataEnsureCapacity(p, capacity); - - size_t size = 0; - const SSysTableMeta* pSysDbTableMeta = NULL; - - getInfosDbMeta(&pSysDbTableMeta, &size); - p->info.rows = buildDbTableInfoBlock(pInfo->sysInfo, p, pSysDbTableMeta, size, TSDB_INFORMATION_SCHEMA_DB); - - getPerfDbMeta(&pSysDbTableMeta, &size); - p->info.rows = buildDbTableInfoBlock(pInfo->sysInfo, p, pSysDbTableMeta, size, TSDB_PERFORMANCE_SCHEMA_DB); - - pInfo->pRes->info.rows = p->info.rows; - relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); - blockDataDestroy(p); - - return pInfo->pRes->info.rows; -} - -int32_t buildDbTableInfoBlock(bool sysInfo, const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size, - const char* dbName) { - char n[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - int32_t numOfRows = p->info.rows; - - for (int32_t i = 0; i < size; ++i) { - const SSysTableMeta* pm = &pSysDbTableMeta[i]; - if (!sysInfo && pm->sysInfo) { - continue; - } - - SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); - - STR_TO_VARSTR(n, pm->name); - colDataAppend(pColInfoData, numOfRows, n, false); - - // database name - STR_TO_VARSTR(n, dbName); - pColInfoData = taosArrayGet(p->pDataBlock, 1); - colDataAppend(pColInfoData, numOfRows, n, false); - - // create time - pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppendNULL(pColInfoData, numOfRows); - - // number of columns - pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*)&pm->colNum, false); - - for (int32_t j = 4; j <= 8; ++j) { - pColInfoData = taosArrayGet(p->pDataBlock, j); - colDataAppendNULL(pColInfoData, numOfRows); - } - - STR_TO_VARSTR(n, "SYSTEM_TABLE"); - - pColInfoData = taosArrayGet(p->pDataBlock, 9); - colDataAppend(pColInfoData, numOfRows, n, false); - - numOfRows += 1; - } - - return numOfRows; -} - -SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, - const char* pUser, SExecTaskInfo* pTaskInfo) { - SSysTableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SSysTableScanInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - SScanPhysiNode* pScanNode = &pScanPhyNode->scan; - SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc; - - int32_t num = 0; - int32_t code = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &num, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - pInfo->accountId = pScanPhyNode->accountId; - pInfo->pUser = taosMemoryStrDup((void*)pUser); - pInfo->sysInfo = pScanPhyNode->sysInfo; - pInfo->showRewrite = pScanPhyNode->showRewrite; - pInfo->pRes = createResDataBlock(pDescNode); - - pInfo->pCondition = pScanNode->node.pConditions; - code = filterInitFromNode(pScanNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - initResultSizeInfo(&pOperator->resultInfo, 4096); - blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - - tNameAssign(&pInfo->name, &pScanNode->tableName); - const char* name = tNameGetTableName(&pInfo->name); - - if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || - strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { - pInfo->readHandle = *(SReadHandle*)readHandle; - } else { - tsem_init(&pInfo->ready, 0, 0); - pInfo->epSet = pScanPhyNode->mgmtEpSet; - pInfo->readHandle = *(SReadHandle*)readHandle; - } - - setOperatorInfo(pOperator, "SysTableScanOperator", QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, false, OP_NOT_OPENED, - pInfo, pTaskInfo); - pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, NULL); - return pOperator; - -_error: - if (pInfo != NULL) { - destroySysScanOperator(pInfo); - } - taosMemoryFreeClear(pOperator); - pTaskInfo->code = code; - return NULL; -} - static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -4271,7 +2578,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { metaReaderInit(&mr, pInfo->readHandle.meta, 0); while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) { - STableKeyInfo* item = tableListGetInfo(pInfo->pTableList, pInfo->curPos); + STableKeyInfo* item = tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->curPos); int32_t code = metaGetTableEntryByUid(&mr, item->uid); tDecoderClear(&mr.coder); if (code != TSDB_CODE_SUCCESS) { @@ -4358,7 +2665,6 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi goto _error; } - pInfo->pTableList = pTableListInfo; pInfo->pRes = createResDataBlock(pDescNode); pInfo->readHandle = *pReadHandle; pInfo->curPos = 0; diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 14e3163455..0efa27dd46 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -17,6 +17,18 @@ #include "executorimpl.h" #include "tdatablock.h" +typedef struct SSortOperatorInfo { + SOptrBasicInfo binfo; + uint32_t sortBufSize; // max buffer size for in-memory sort + SArray* pSortInfo; + SSortHandle* pSortHandle; + SColMatchInfo matchInfo; + int32_t bufPageSize; + int64_t startTs; // sort start time + uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included. + SLimitInfo limitInfo; +} SSortOperatorInfo; + static SSDataBlock* doSort(SOperatorInfo* pOperator); static int32_t doOpenSortOperator(SOperatorInfo* pOperator); static int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len); diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c new file mode 100644 index 0000000000..f6a0c57b66 --- /dev/null +++ b/source/libs/executor/src/sysscanoperator.c @@ -0,0 +1,1774 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "executorimpl.h" +#include "filter.h" +#include "function.h" +#include "functionMgt.h" +#include "os.h" +#include "querynodes.h" +#include "systable.h" +#include "tname.h" +#include "ttime.h" + +#include "tdatablock.h" +#include "tmsg.h" + +#include "query.h" +#include "tcompare.h" +#include "thash.h" +#include "ttypes.h" +#include "vnode.h" + +typedef int (*__optSysFilter)(void* a, void* b, int16_t dtype); +typedef int32_t (*__sys_filte)(void* pMeta, SNode* cond, SArray* result); +typedef int32_t (*__sys_check)(SNode* cond); + +typedef struct SSTabFltArg { + void* pMeta; + void* pVnode; +} SSTabFltArg; + +typedef struct SSysTableIndex { + int8_t init; + SArray* uids; + int32_t lastIdx; +} SSysTableIndex; + +typedef struct SSysTableScanInfo { + SRetrieveMetaTableRsp* pRsp; + SRetrieveTableReq req; + SEpSet epSet; + tsem_t ready; + SReadHandle readHandle; + int32_t accountId; + const char* pUser; + bool sysInfo; + bool showRewrite; + SNode* pCondition; // db_name filter condition, to discard data that are not in current database + SMTbCursor* pCur; // cursor for iterate the local table meta store. + SSysTableIndex* pIdx; // idx for local table meta + SColMatchInfo matchInfo; + SName name; + SSDataBlock* pRes; + int64_t numOfBlocks; // extract basic running information. + SLoadRemoteDataInfo loadInfo; +} SSysTableScanInfo; + +typedef struct { + const char* name; + __sys_check chkFunc; + __sys_filte fltFunc; +} SSTabFltFuncDef; + +typedef struct MergeIndex { + int idx; + int len; +} MergeIndex; + +static int32_t sysChkFilter__Comm(SNode* pNode); +static int32_t sysChkFilter__DBName(SNode* pNode); +static int32_t sysChkFilter__VgroupId(SNode* pNode); +static int32_t sysChkFilter__TableName(SNode* pNode); +static int32_t sysChkFilter__CreateTime(SNode* pNode); +static int32_t sysChkFilter__Ncolumn(SNode* pNode); +static int32_t sysChkFilter__Ttl(SNode* pNode); +static int32_t sysChkFilter__STableName(SNode* pNode); +static int32_t sysChkFilter__Uid(SNode* pNode); +static int32_t sysChkFilter__Type(SNode* pNode); + +static int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result); +static int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result); +static int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result); +static int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result); +static int32_t sysFilte__Ncolumn(void* arg, SNode* pNode, SArray* result); +static int32_t sysFilte__Ttl(void* arg, SNode* pNode, SArray* result); +static int32_t sysFilte__STableName(void* arg, SNode* pNode, SArray* result); +static int32_t sysFilte__Uid(void* arg, SNode* pNode, SArray* result); +static int32_t sysFilte__Type(void* arg, SNode* pNode, SArray* result); + +const SSTabFltFuncDef filterDict[] = { + {.name = "table_name", .chkFunc = sysChkFilter__TableName, .fltFunc = sysFilte__TableName}, + {.name = "db_name", .chkFunc = sysChkFilter__DBName, .fltFunc = sysFilte__DbName}, + {.name = "create_time", .chkFunc = sysChkFilter__CreateTime, .fltFunc = sysFilte__CreateTime}, + {.name = "columns", .chkFunc = sysChkFilter__Ncolumn, .fltFunc = sysFilte__Ncolumn}, + {.name = "ttl", .chkFunc = sysChkFilter__Ttl, .fltFunc = sysFilte__Ttl}, + {.name = "stable_name", .chkFunc = sysChkFilter__STableName, .fltFunc = sysFilte__STableName}, + {.name = "vgroup_id", .chkFunc = sysChkFilter__VgroupId, .fltFunc = sysFilte__VgroupId}, + {.name = "uid", .chkFunc = sysChkFilter__Uid, .fltFunc = sysFilte__Uid}, + {.name = "type", .chkFunc = sysChkFilter__Type, .fltFunc = sysFilte__Type}}; + +#define SYSTAB_FILTER_DICT_SIZE (sizeof(filterDict) / sizeof(filterDict[0])) + +static int32_t buildDbTableInfoBlock(bool sysInfo, const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, + size_t size, const char* dbName); + +static char* SYSTABLE_IDX_COLUMN[] = {"table_name", "db_name", "create_time", "columns", + "ttl", "stable_name", "vgroup_id', 'uid", "type"}; + +static char* SYSTABLE_SPECIAL_COL[] = {"db_name", "vgroup_id"}; + +static int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity); +static SSDataBlock* buildInfoSchemaTableMetaBlock(char* tableName); +static void destroySysScanOperator(void* param); +static int32_t loadSysTableCallback(void* param, SDataBuf* pMsg, int32_t code); +static SSDataBlock* doFilterResult(SSDataBlock* pDataBlock, SFilterInfo* pFilterInfo); +static __optSysFilter optSysGetFilterFunc(int32_t ctype, bool* reverse); + +static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrSuperTable, + SMetaReader* smrChildTable, const char* dbname, const char* tableName, + int32_t* pNumOfRows, const SSDataBlock* dataBlock); + +static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t numOfRows, SSDataBlock* dataBlock, + SFilterInfo* pFilterInfo); + +int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result) { + void* pVnode = ((SSTabFltArg*)arg)->pVnode; + + const char* db = NULL; + vnodeGetInfo(pVnode, &db, NULL); + + SName sn = {0}; + char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); + + tNameGetDbName(&sn, varDataVal(dbname)); + varDataSetLen(dbname, strlen(varDataVal(dbname))); + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + + bool reverse = false; + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + + int ret = func(dbname, pVal->datum.p, TSDB_DATA_TYPE_VARCHAR); + if (ret == 0) return 0; + + return -2; +} + +int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result) { + void* pVnode = ((SSTabFltArg*)arg)->pVnode; + + int64_t vgId = 0; + vnodeGetInfo(pVnode, NULL, (int32_t*)&vgId); + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + + bool reverse = false; + + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + + int ret = func(&vgId, &pVal->datum.i, TSDB_DATA_TYPE_BIGINT); + if (ret == 0) return 0; + + return -1; +} + +int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result) { + void* pMeta = ((SSTabFltArg*)arg)->pMeta; + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + bool reverse = false; + + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + + SMetaFltParam param = {.suid = 0, + .cid = 0, + .type = TSDB_DATA_TYPE_VARCHAR, + .val = pVal->datum.p, + .reverse = reverse, + .filterFunc = func}; + return -1; +} + +int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result) { + void* pMeta = ((SSTabFltArg*)arg)->pMeta; + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + bool reverse = false; + + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + + SMetaFltParam param = {.suid = 0, + .cid = 0, + .type = TSDB_DATA_TYPE_BIGINT, + .val = &pVal->datum.i, + .reverse = reverse, + .filterFunc = func}; + + int32_t ret = metaFilterCreateTime(pMeta, ¶m, result); + return ret; +} + +int32_t sysFilte__Ncolumn(void* arg, SNode* pNode, SArray* result) { + void* pMeta = ((SSTabFltArg*)arg)->pMeta; + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + bool reverse = false; + + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + return -1; +} + +int32_t sysFilte__Ttl(void* arg, SNode* pNode, SArray* result) { + void* pMeta = ((SSTabFltArg*)arg)->pMeta; + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + bool reverse = false; + + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + return -1; +} + +int32_t sysFilte__STableName(void* arg, SNode* pNode, SArray* result) { + void* pMeta = ((SSTabFltArg*)arg)->pMeta; + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + bool reverse = false; + + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + return -1; +} + +int32_t sysFilte__Uid(void* arg, SNode* pNode, SArray* result) { + void* pMeta = ((SSTabFltArg*)arg)->pMeta; + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + bool reverse = false; + + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + return -1; +} + +int32_t sysFilte__Type(void* arg, SNode* pNode, SArray* result) { + void* pMeta = ((SSTabFltArg*)arg)->pMeta; + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + bool reverse = false; + + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + return -1; +} + +int optSysDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) { + int32_t cmp = func(a, b); + switch (comparType) { + case OP_TYPE_LOWER_THAN: + if (cmp < 0) return 0; + break; + case OP_TYPE_LOWER_EQUAL: { + if (cmp <= 0) return 0; + break; + } + case OP_TYPE_GREATER_THAN: { + if (cmp > 0) return 0; + break; + } + case OP_TYPE_GREATER_EQUAL: { + if (cmp >= 0) return 0; + break; + } + case OP_TYPE_EQUAL: { + if (cmp == 0) return 0; + break; + } + default: + return -1; + } + return cmp; +} + +static int optSysFilterFuncImpl__LowerThan(void* a, void* b, int16_t dtype) { + __compar_fn_t func = getComparFunc(dtype, 0); + return optSysDoCompare(func, OP_TYPE_LOWER_THAN, a, b); +} +static int optSysFilterFuncImpl__LowerEqual(void* a, void* b, int16_t dtype) { + __compar_fn_t func = getComparFunc(dtype, 0); + return optSysDoCompare(func, OP_TYPE_LOWER_EQUAL, a, b); +} +static int optSysFilterFuncImpl__GreaterThan(void* a, void* b, int16_t dtype) { + __compar_fn_t func = getComparFunc(dtype, 0); + return optSysDoCompare(func, OP_TYPE_GREATER_THAN, a, b); +} +static int optSysFilterFuncImpl__GreaterEqual(void* a, void* b, int16_t dtype) { + __compar_fn_t func = getComparFunc(dtype, 0); + return optSysDoCompare(func, OP_TYPE_GREATER_EQUAL, a, b); +} +static int optSysFilterFuncImpl__Equal(void* a, void* b, int16_t dtype) { + __compar_fn_t func = getComparFunc(dtype, 0); + return optSysDoCompare(func, OP_TYPE_EQUAL, a, b); +} + +static int optSysFilterFuncImpl__NoEqual(void* a, void* b, int16_t dtype) { + __compar_fn_t func = getComparFunc(dtype, 0); + return optSysDoCompare(func, OP_TYPE_NOT_EQUAL, a, b); +} + +static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result); +static int32_t optSysTabFilteImpl(void* arg, SNode* cond, SArray* result); +static int32_t optSysCheckOper(SNode* pOpear); +static int32_t optSysMergeRslt(SArray* mRslt, SArray* rslt); + +__optSysFilter optSysGetFilterFunc(int32_t ctype, bool* reverse) { + if (ctype == OP_TYPE_LOWER_EQUAL || ctype == OP_TYPE_LOWER_THAN) { + *reverse = true; + } + if (ctype == OP_TYPE_LOWER_THAN) + return optSysFilterFuncImpl__LowerThan; + else if (ctype == OP_TYPE_LOWER_EQUAL) + return optSysFilterFuncImpl__LowerEqual; + else if (ctype == OP_TYPE_GREATER_THAN) + return optSysFilterFuncImpl__GreaterThan; + else if (ctype == OP_TYPE_GREATER_EQUAL) + return optSysFilterFuncImpl__GreaterEqual; + else if (ctype == OP_TYPE_EQUAL) + return optSysFilterFuncImpl__Equal; + else if (ctype == OP_TYPE_NOT_EQUAL) + return optSysFilterFuncImpl__NoEqual; + return NULL; +} + +static bool sysTableIsOperatorCondOnOneTable(SNode* pCond, char* condTable) { + SOperatorNode* node = (SOperatorNode*)pCond; + if (node->opType == OP_TYPE_EQUAL) { + if (nodeType(node->pLeft) == QUERY_NODE_COLUMN && + strcasecmp(nodesGetNameFromColumnNode(node->pLeft), "table_name") == 0 && + nodeType(node->pRight) == QUERY_NODE_VALUE) { + SValueNode* pValue = (SValueNode*)node->pRight; + if (pValue->node.resType.type == TSDB_DATA_TYPE_NCHAR || pValue->node.resType.type == TSDB_DATA_TYPE_VARCHAR || + pValue->node.resType.type == TSDB_DATA_TYPE_BINARY) { + char* value = nodesGetValueFromNode(pValue); + strncpy(condTable, varDataVal(value), TSDB_TABLE_NAME_LEN); + return true; + } + } + } + return false; +} + +static bool sysTableIsCondOnOneTable(SNode* pCond, char* condTable) { + if (pCond == NULL) { + return false; + } + if (nodeType(pCond) == QUERY_NODE_LOGIC_CONDITION) { + SLogicConditionNode* node = (SLogicConditionNode*)pCond; + if (LOGIC_COND_TYPE_AND == node->condType) { + SNode* pChild = NULL; + FOREACH(pChild, node->pParameterList) { + if (QUERY_NODE_OPERATOR == nodeType(pChild) && sysTableIsOperatorCondOnOneTable(pChild, condTable)) { + return true; + } + } + } + } + + if (QUERY_NODE_OPERATOR == nodeType(pCond)) { + return sysTableIsOperatorCondOnOneTable(pCond, condTable); + } + + return false; +} + +static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSysTableScanInfo* pInfo = pOperator->info; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + blockDataCleanup(pInfo->pRes); + int32_t numOfRows = 0; + + SSDataBlock* dataBlock = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TAGS); + blockDataEnsureCapacity(dataBlock, pOperator->resultInfo.capacity); + + const char* db = NULL; + int32_t vgId = 0; + vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + + SName sn = {0}; + char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); + + tNameGetDbName(&sn, varDataVal(dbname)); + varDataSetLen(dbname, strlen(varDataVal(dbname))); + + char condTableName[TSDB_TABLE_NAME_LEN] = {0}; + // optimize when sql like where table_name='tablename' and xxx. + if (pInfo->pCondition && sysTableIsCondOnOneTable(pInfo->pCondition, condTableName)) { + char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(tableName, condTableName); + + SMetaReader smrChildTable = {0}; + metaReaderInit(&smrChildTable, pInfo->readHandle.meta, 0); + int32_t code = metaGetTableEntryByName(&smrChildTable, condTableName); + if (code != TSDB_CODE_SUCCESS) { + // terrno has been set by metaGetTableEntryByName, therefore, return directly + return NULL; + } + + if (smrChildTable.me.type != TSDB_CHILD_TABLE) { + metaReaderClear(&smrChildTable); + blockDataDestroy(dataBlock); + pInfo->loadInfo.totalRows = 0; + return NULL; + } + + SMetaReader smrSuperTable = {0}; + metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, META_READER_NOLOCK); + code = metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); + if (code != TSDB_CODE_SUCCESS) { + // terrno has been set by metaGetTableEntryByUid + return NULL; + } + + sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock); + metaReaderClear(&smrSuperTable); + metaReaderClear(&smrChildTable); + if (numOfRows > 0) { + relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); + numOfRows = 0; + } + blockDataDestroy(dataBlock); + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + setOperatorCompleted(pOperator); + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; + } + + int32_t ret = 0; + if (pInfo->pCur == NULL) { + pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); + } + + while ((ret = metaTbCursorNext(pInfo->pCur)) == 0) { + if (pInfo->pCur->mr.me.type != TSDB_CHILD_TABLE) { + continue; + } + + char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); + + SMetaReader smrSuperTable = {0}; + metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0); + uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; + int32_t code = metaGetTableEntryByUid(&smrSuperTable, suid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), + GET_TASKID(pTaskInfo)); + metaReaderClear(&smrSuperTable); + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + T_LONG_JMP(pTaskInfo->env, terrno); + } + + sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows, dataBlock); + + metaReaderClear(&smrSuperTable); + + if (numOfRows >= pOperator->resultInfo.capacity) { + relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); + numOfRows = 0; + + if (pInfo->pRes->info.rows > 0) { + break; + } + } + } + + if (numOfRows > 0) { + relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); + numOfRows = 0; + } + + blockDataDestroy(dataBlock); + if (ret != 0) { + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + setOperatorCompleted(pOperator); + } + + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; +} + +void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t numOfRows, SSDataBlock* dataBlock, + SFilterInfo* pFilterInfo) { + dataBlock->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, dataBlock->pDataBlock, false); + doFilterResult(pInfo->pRes, pFilterInfo); + blockDataCleanup(dataBlock); +} + +int32_t convertTagDataToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len) { + int32_t n = 0; + + switch (type) { + case TSDB_DATA_TYPE_NULL: + n = sprintf(str, "null"); + break; + + case TSDB_DATA_TYPE_BOOL: + n = sprintf(str, (*(int8_t*)buf) ? "true" : "false"); + break; + + case TSDB_DATA_TYPE_TINYINT: + n = sprintf(str, "%d", *(int8_t*)buf); + break; + + case TSDB_DATA_TYPE_SMALLINT: + n = sprintf(str, "%d", *(int16_t*)buf); + break; + + case TSDB_DATA_TYPE_INT: + n = sprintf(str, "%d", *(int32_t*)buf); + break; + + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + n = sprintf(str, "%" PRId64, *(int64_t*)buf); + break; + + case TSDB_DATA_TYPE_FLOAT: + n = sprintf(str, "%.5f", GET_FLOAT_VAL(buf)); + break; + + case TSDB_DATA_TYPE_DOUBLE: + n = sprintf(str, "%.9f", GET_DOUBLE_VAL(buf)); + break; + + case TSDB_DATA_TYPE_BINARY: + if (bufSize < 0) { + return TSDB_CODE_TSC_INVALID_VALUE; + } + + memcpy(str, buf, bufSize); + n = bufSize; + break; + case TSDB_DATA_TYPE_NCHAR: + if (bufSize < 0) { + return TSDB_CODE_TSC_INVALID_VALUE; + } + + int32_t length = taosUcs4ToMbs((TdUcs4*)buf, bufSize, str); + if (length <= 0) { + return TSDB_CODE_TSC_INVALID_VALUE; + } + n = length; + break; + case TSDB_DATA_TYPE_UTINYINT: + n = sprintf(str, "%u", *(uint8_t*)buf); + break; + + case TSDB_DATA_TYPE_USMALLINT: + n = sprintf(str, "%u", *(uint16_t*)buf); + break; + + case TSDB_DATA_TYPE_UINT: + n = sprintf(str, "%u", *(uint32_t*)buf); + break; + + case TSDB_DATA_TYPE_UBIGINT: + n = sprintf(str, "%" PRIu64, *(uint64_t*)buf); + break; + + default: + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (len) *len = n; + + return TSDB_CODE_SUCCESS; +} + +static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrSuperTable, + SMetaReader* smrChildTable, const char* dbname, const char* tableName, + int32_t* pNumOfRows, const SSDataBlock* dataBlock) { + char stableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(stableName, (*smrSuperTable).me.name); + + int32_t numOfRows = *pNumOfRows; + + int32_t numOfTags = (*smrSuperTable).me.stbEntry.schemaTag.nCols; + for (int32_t i = 0; i < numOfTags; ++i) { + SColumnInfoData* pColInfoData = NULL; + + // table name + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 0); + colDataAppend(pColInfoData, numOfRows, tableName, false); + + // database name + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 1); + colDataAppend(pColInfoData, numOfRows, dbname, false); + + // super table name + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, stableName, false); + + // tag name + char tagName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(tagName, (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].name); + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, tagName, false); + + // tag type + int8_t tagType = (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].type; + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 4); + char tagTypeStr[VARSTR_HEADER_SIZE + 32]; + int tagTypeLen = sprintf(varDataVal(tagTypeStr), "%s", tDataTypes[tagType].name); + if (tagType == TSDB_DATA_TYPE_VARCHAR) { + tagTypeLen += sprintf(varDataVal(tagTypeStr) + tagTypeLen, "(%d)", + (int32_t)((*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE)); + } else if (tagType == TSDB_DATA_TYPE_NCHAR) { + tagTypeLen += sprintf( + varDataVal(tagTypeStr) + tagTypeLen, "(%d)", + (int32_t)(((*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); + } + varDataSetLen(tagTypeStr, tagTypeLen); + colDataAppend(pColInfoData, numOfRows, (char*)tagTypeStr, false); + + STagVal tagVal = {0}; + tagVal.cid = (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].colId; + char* tagData = NULL; + uint32_t tagLen = 0; + + if (tagType == TSDB_DATA_TYPE_JSON) { + tagData = (char*)smrChildTable->me.ctbEntry.pTags; + } else { + bool exist = tTagGet((STag*)smrChildTable->me.ctbEntry.pTags, &tagVal); + if (exist) { + if (IS_VAR_DATA_TYPE(tagType)) { + tagData = (char*)tagVal.pData; + tagLen = tagVal.nData; + } else { + tagData = (char*)&tagVal.i64; + tagLen = tDataTypes[tagType].bytes; + } + } + } + + char* tagVarChar = NULL; + if (tagData != NULL) { + if (tagType == TSDB_DATA_TYPE_JSON) { + char* tagJson = parseTagDatatoJson(tagData); + tagVarChar = taosMemoryMalloc(strlen(tagJson) + VARSTR_HEADER_SIZE); + memcpy(varDataVal(tagVarChar), tagJson, strlen(tagJson)); + varDataSetLen(tagVarChar, strlen(tagJson)); + taosMemoryFree(tagJson); + } else { + int32_t bufSize = IS_VAR_DATA_TYPE(tagType) ? (tagLen + VARSTR_HEADER_SIZE) + : (3 + DBL_MANT_DIG - DBL_MIN_EXP + VARSTR_HEADER_SIZE); + tagVarChar = taosMemoryMalloc(bufSize); + int32_t len = -1; + convertTagDataToStr(varDataVal(tagVarChar), tagType, tagData, tagLen, &len); + varDataSetLen(tagVarChar, len); + } + } + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, tagVarChar, + (tagData == NULL) || (tagType == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(tagData))); + taosMemoryFree(tagVarChar); + ++numOfRows; + } + + *pNumOfRows = numOfRows; + + return TSDB_CODE_SUCCESS; +} + +static SSDataBlock* buildInfoSchemaTableMetaBlock(char* tableName) { + size_t size = 0; + const SSysTableMeta* pMeta = NULL; + getInfosDbMeta(&pMeta, &size); + + int32_t index = 0; + for (int32_t i = 0; i < size; ++i) { + if (strcmp(pMeta[i].name, tableName) == 0) { + index = i; + break; + } + } + + SSDataBlock* pBlock = createDataBlock(); + for (int32_t i = 0; i < pMeta[index].colNum; ++i) { + SColumnInfoData colInfoData = + createColumnInfoData(pMeta[index].schema[i].type, pMeta[index].schema[i].bytes, i + 1); + blockDataAppendColInfo(pBlock, &colInfoData); + } + + return pBlock; +} + +int32_t buildDbTableInfoBlock(bool sysInfo, const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size, + const char* dbName) { + char n[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + int32_t numOfRows = p->info.rows; + + for (int32_t i = 0; i < size; ++i) { + const SSysTableMeta* pm = &pSysDbTableMeta[i]; + if (!sysInfo && pm->sysInfo) { + continue; + } + + SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); + + STR_TO_VARSTR(n, pm->name); + colDataAppend(pColInfoData, numOfRows, n, false); + + // database name + STR_TO_VARSTR(n, dbName); + pColInfoData = taosArrayGet(p->pDataBlock, 1); + colDataAppend(pColInfoData, numOfRows, n, false); + + // create time + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppendNULL(pColInfoData, numOfRows); + + // number of columns + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, (char*)&pm->colNum, false); + + for (int32_t j = 4; j <= 8; ++j) { + pColInfoData = taosArrayGet(p->pDataBlock, j); + colDataAppendNULL(pColInfoData, numOfRows); + } + + STR_TO_VARSTR(n, "SYSTEM_TABLE"); + + pColInfoData = taosArrayGet(p->pDataBlock, 9); + colDataAppend(pColInfoData, numOfRows, n, false); + + numOfRows += 1; + } + + return numOfRows; +} + +int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity) { + SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TABLES); + blockDataEnsureCapacity(p, capacity); + + size_t size = 0; + const SSysTableMeta* pSysDbTableMeta = NULL; + + getInfosDbMeta(&pSysDbTableMeta, &size); + p->info.rows = buildDbTableInfoBlock(pInfo->sysInfo, p, pSysDbTableMeta, size, TSDB_INFORMATION_SCHEMA_DB); + + getPerfDbMeta(&pSysDbTableMeta, &size); + p->info.rows = buildDbTableInfoBlock(pInfo->sysInfo, p, pSysDbTableMeta, size, TSDB_PERFORMANCE_SCHEMA_DB); + + pInfo->pRes->info.rows = p->info.rows; + relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); + blockDataDestroy(p); + + return pInfo->pRes->info.rows; +} + +static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSysTableScanInfo* pInfo = pOperator->info; + + SSysTableIndex* pIdx = pInfo->pIdx; + blockDataCleanup(pInfo->pRes); + int32_t numOfRows = 0; + + int ret = 0; + + const char* db = NULL; + int32_t vgId = 0; + vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + + SName sn = {0}; + char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); + + tNameGetDbName(&sn, varDataVal(dbname)); + varDataSetLen(dbname, strlen(varDataVal(dbname))); + + SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TABLES); + blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); + + char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + int32_t i = pIdx->lastIdx; + for (; i < taosArrayGetSize(pIdx->uids); i++) { + tb_uid_t* uid = taosArrayGet(pIdx->uids, i); + + SMetaReader mr = {0}; + metaReaderInit(&mr, pInfo->readHandle.meta, 0); + ret = metaGetTableEntryByUid(&mr, *uid); + if (ret < 0) { + metaReaderClear(&mr); + continue; + } + STR_TO_VARSTR(n, mr.me.name); + + // table name + SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); + colDataAppend(pColInfoData, numOfRows, n, false); + + // database name + pColInfoData = taosArrayGet(p->pDataBlock, 1); + colDataAppend(pColInfoData, numOfRows, dbname, false); + + // vgId + pColInfoData = taosArrayGet(p->pDataBlock, 6); + colDataAppend(pColInfoData, numOfRows, (char*)&vgId, false); + + int32_t tableType = mr.me.type; + if (tableType == TSDB_CHILD_TABLE) { + // create time + int64_t ts = mr.me.ctbEntry.ctime; + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, (char*)&ts, false); + + SMetaReader mr1 = {0}; + metaReaderInit(&mr1, pInfo->readHandle.meta, META_READER_NOLOCK); + + int64_t suid = mr.me.ctbEntry.suid; + int32_t code = metaGetTableEntryByUid(&mr1, suid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name, + suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); + metaReaderClear(&mr1); + metaReaderClear(&mr); + T_LONG_JMP(pTaskInfo->env, terrno); + } + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, (char*)&mr1.me.stbEntry.schemaRow.nCols, false); + + // super table name + STR_TO_VARSTR(n, mr1.me.name); + pColInfoData = taosArrayGet(p->pDataBlock, 4); + colDataAppend(pColInfoData, numOfRows, n, false); + metaReaderClear(&mr1); + + // table comment + pColInfoData = taosArrayGet(p->pDataBlock, 8); + if (mr.me.ctbEntry.commentLen > 0) { + char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, mr.me.ctbEntry.comment); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else if (mr.me.ctbEntry.commentLen == 0) { + char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, ""); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else { + colDataAppendNULL(pColInfoData, numOfRows); + } + + // uid + pColInfoData = taosArrayGet(p->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.uid, false); + + // ttl + pColInfoData = taosArrayGet(p->pDataBlock, 7); + colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.ctbEntry.ttlDays, false); + + STR_TO_VARSTR(n, "CHILD_TABLE"); + + } else if (tableType == TSDB_NORMAL_TABLE) { + // create time + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false); + + // number of columns + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.schemaRow.nCols, false); + + // super table name + pColInfoData = taosArrayGet(p->pDataBlock, 4); + colDataAppendNULL(pColInfoData, numOfRows); + + // table comment + pColInfoData = taosArrayGet(p->pDataBlock, 8); + if (mr.me.ntbEntry.commentLen > 0) { + char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, mr.me.ntbEntry.comment); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else if (mr.me.ntbEntry.commentLen == 0) { + char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, ""); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else { + colDataAppendNULL(pColInfoData, numOfRows); + } + + // uid + pColInfoData = taosArrayGet(p->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.uid, false); + + // ttl + pColInfoData = taosArrayGet(p->pDataBlock, 7); + colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.ntbEntry.ttlDays, false); + + STR_TO_VARSTR(n, "NORMAL_TABLE"); + // impl later + } + + metaReaderClear(&mr); + + pColInfoData = taosArrayGet(p->pDataBlock, 9); + colDataAppend(pColInfoData, numOfRows, n, false); + + if (++numOfRows >= pOperator->resultInfo.capacity) { + p->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); + doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); + + blockDataCleanup(p); + numOfRows = 0; + + if (pInfo->pRes->info.rows > 0) { + break; + } + } + } + + if (numOfRows > 0) { + p->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); + doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); + + blockDataCleanup(p); + numOfRows = 0; + } + + if (i >= taosArrayGetSize(pIdx->uids)) { + setOperatorCompleted(pOperator); + } else { + pIdx->lastIdx = i + 1; + } + + blockDataDestroy(p); + + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; +} + +static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + SSysTableScanInfo* pInfo = pOperator->info; + if (pInfo->pCur == NULL) { + pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); + } + + blockDataCleanup(pInfo->pRes); + int32_t numOfRows = 0; + + const char* db = NULL; + int32_t vgId = 0; + vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + + SName sn = {0}; + char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); + + tNameGetDbName(&sn, varDataVal(dbname)); + varDataSetLen(dbname, strlen(varDataVal(dbname))); + + SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TABLES); + blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); + + char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + + int32_t ret = 0; + while ((ret = metaTbCursorNext(pInfo->pCur)) == 0) { + STR_TO_VARSTR(n, pInfo->pCur->mr.me.name); + + // table name + SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); + colDataAppend(pColInfoData, numOfRows, n, false); + + // database name + pColInfoData = taosArrayGet(p->pDataBlock, 1); + colDataAppend(pColInfoData, numOfRows, dbname, false); + + // vgId + pColInfoData = taosArrayGet(p->pDataBlock, 6); + colDataAppend(pColInfoData, numOfRows, (char*)&vgId, false); + + int32_t tableType = pInfo->pCur->mr.me.type; + if (tableType == TSDB_CHILD_TABLE) { + // create time + int64_t ts = pInfo->pCur->mr.me.ctbEntry.ctime; + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, (char*)&ts, false); + + SMetaReader mr = {0}; + metaReaderInit(&mr, pInfo->readHandle.meta, META_READER_NOLOCK); + + uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; + int32_t code = metaGetTableEntryByUid(&mr, suid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name, + suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); + metaReaderClear(&mr); + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + T_LONG_JMP(pTaskInfo->env, terrno); + } + + // number of columns + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.stbEntry.schemaRow.nCols, false); + + // super table name + STR_TO_VARSTR(n, mr.me.name); + pColInfoData = taosArrayGet(p->pDataBlock, 4); + colDataAppend(pColInfoData, numOfRows, n, false); + metaReaderClear(&mr); + + // table comment + pColInfoData = taosArrayGet(p->pDataBlock, 8); + if (pInfo->pCur->mr.me.ctbEntry.commentLen > 0) { + char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ctbEntry.comment); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else if (pInfo->pCur->mr.me.ctbEntry.commentLen == 0) { + char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, ""); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else { + colDataAppendNULL(pColInfoData, numOfRows); + } + + // uid + pColInfoData = taosArrayGet(p->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); + + // ttl + pColInfoData = taosArrayGet(p->pDataBlock, 7); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ctbEntry.ttlDays, false); + + STR_TO_VARSTR(n, "CHILD_TABLE"); + } else if (tableType == TSDB_NORMAL_TABLE) { + // create time + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false); + + // number of columns + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.schemaRow.nCols, false); + + // super table name + pColInfoData = taosArrayGet(p->pDataBlock, 4); + colDataAppendNULL(pColInfoData, numOfRows); + + // table comment + pColInfoData = taosArrayGet(p->pDataBlock, 8); + if (pInfo->pCur->mr.me.ntbEntry.commentLen > 0) { + char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ntbEntry.comment); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else if (pInfo->pCur->mr.me.ntbEntry.commentLen == 0) { + char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, ""); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else { + colDataAppendNULL(pColInfoData, numOfRows); + } + + // uid + pColInfoData = taosArrayGet(p->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); + + // ttl + pColInfoData = taosArrayGet(p->pDataBlock, 7); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ttlDays, false); + + STR_TO_VARSTR(n, "NORMAL_TABLE"); + } + + pColInfoData = taosArrayGet(p->pDataBlock, 9); + colDataAppend(pColInfoData, numOfRows, n, false); + + if (++numOfRows >= pOperator->resultInfo.capacity) { + p->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); + doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); + + blockDataCleanup(p); + numOfRows = 0; + + if (pInfo->pRes->info.rows > 0) { + break; + } + } + } + + if (numOfRows > 0) { + p->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); + doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); + + blockDataCleanup(p); + numOfRows = 0; + } + + blockDataDestroy(p); + + // todo temporarily free the cursor here, the true reason why the free is not valid needs to be found + if (ret != 0) { + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + setOperatorCompleted(pOperator); + } + + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; +} + +static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSysTableScanInfo* pInfo = pOperator->info; + + SNode* pCondition = pInfo->pCondition; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + // the retrieve is executed on the mnode, so return tables that belongs to the information schema database. + if (pInfo->readHandle.mnd != NULL) { + buildSysDbTableInfo(pInfo, pOperator->resultInfo.capacity); + doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + + setOperatorCompleted(pOperator); + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; + } else { + if (pInfo->showRewrite == false) { + if (pCondition != NULL && pInfo->pIdx == NULL) { + SSTabFltArg arg = {.pMeta = pInfo->readHandle.meta, .pVnode = pInfo->readHandle.vnode}; + + SSysTableIndex* idx = taosMemoryMalloc(sizeof(SSysTableIndex)); + idx->init = 0; + idx->uids = taosArrayInit(128, sizeof(int64_t)); + idx->lastIdx = 0; + + pInfo->pIdx = idx; // set idx arg + + int flt = optSysTabFilte(&arg, pCondition, idx->uids); + if (flt == 0) { + pInfo->pIdx->init = 1; + SSDataBlock* blk = sysTableBuildUserTablesByUids(pOperator); + return blk; + } else if (flt == -2) { + qDebug("%s failed to get sys table info by idx, empty result", GET_TASKID(pTaskInfo)); + return NULL; + } else if (flt == -1) { + // not idx + qDebug("%s failed to get sys table info by idx, scan sys table one by one", GET_TASKID(pTaskInfo)); + } + } else if (pCondition != NULL && (pInfo->pIdx != NULL && pInfo->pIdx->init == 1)) { + SSDataBlock* blk = sysTableBuildUserTablesByUids(pOperator); + return blk; + } + } + + return sysTableBuildUserTables(pOperator); + } + return NULL; +} + +static SSDataBlock* sysTableScanUserSTables(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSysTableScanInfo* pInfo = pOperator->info; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + pInfo->pRes->info.rows = 0; + pOperator->status = OP_EXEC_DONE; + + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; +} + +static int32_t getSysTableDbNameColId(const char* pTable) { + // if (0 == strcmp(TSDB_INS_TABLE_INDEXES, pTable)) { + // return 1; + // } + return TSDB_INS_USER_STABLES_DBNAME_COLID; +} + +static EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) { + int32_t code = TSDB_CODE_SUCCESS; + ENodeType nType = nodeType(pNode); + + switch (nType) { + case QUERY_NODE_OPERATOR: { + SOperatorNode* node = (SOperatorNode*)pNode; + if (OP_TYPE_EQUAL == node->opType) { + *(int32_t*)pContext = 1; + return DEAL_RES_CONTINUE; + } + + *(int32_t*)pContext = 0; + return DEAL_RES_IGNORE_CHILD; + } + case QUERY_NODE_COLUMN: { + if (1 != *(int32_t*)pContext) { + return DEAL_RES_CONTINUE; + } + + SColumnNode* node = (SColumnNode*)pNode; + if (getSysTableDbNameColId(node->tableName) == node->colId) { + *(int32_t*)pContext = 2; + return DEAL_RES_CONTINUE; + } + + *(int32_t*)pContext = 0; + return DEAL_RES_CONTINUE; + } + case QUERY_NODE_VALUE: { + if (2 != *(int32_t*)pContext) { + return DEAL_RES_CONTINUE; + } + + SValueNode* node = (SValueNode*)pNode; + char* dbName = nodesGetValueFromNode(node); + strncpy(pContext, varDataVal(dbName), varDataLen(dbName)); + *((char*)pContext + varDataLen(dbName)) = 0; + return DEAL_RES_END; // stop walk + } + default: + break; + } + return DEAL_RES_CONTINUE; +} + +static void getDBNameFromCondition(SNode* pCondition, const char* dbName) { + if (NULL == pCondition) { + return; + } + nodesWalkExpr(pCondition, getDBNameFromConditionWalker, (char*)dbName); +} + +static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { + // build message and send to mnode to fetch the content of system tables. + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSysTableScanInfo* pInfo = pOperator->info; + char dbName[TSDB_DB_NAME_LEN] = {0}; + + const char* name = tNameGetTableName(&pInfo->name); + if (pInfo->showRewrite) { + getDBNameFromCondition(pInfo->pCondition, dbName); + sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName); + } + + if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { + return sysTableScanUserTables(pOperator); + } else if (strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { + return sysTableScanUserTags(pOperator); + } else if (strncasecmp(name, TSDB_INS_TABLE_STABLES, TSDB_TABLE_FNAME_LEN) == 0 && pInfo->showRewrite && + IS_SYS_DBNAME(dbName)) { + return sysTableScanUserSTables(pOperator); + } else { // load the meta from mnode of the given epset + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + while (1) { + int64_t startTs = taosGetTimestampUs(); + tstrncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); + tstrncpy(pInfo->req.user, pInfo->pUser, tListLen(pInfo->req.user)); + + int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req); + char* buf1 = taosMemoryCalloc(1, contLen); + tSerializeSRetrieveTableReq(buf1, contLen, &pInfo->req); + + // send the fetch remote task result reques + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (NULL == pMsgSendInfo) { + qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + + int32_t msgType = (strcasecmp(name, TSDB_INS_TABLE_DNODE_VARIABLES) == 0) ? TDMT_DND_SYSTABLE_RETRIEVE + : TDMT_MND_SYSTABLE_RETRIEVE; + + pMsgSendInfo->param = pOperator; + pMsgSendInfo->msgInfo.pData = buf1; + pMsgSendInfo->msgInfo.len = contLen; + pMsgSendInfo->msgType = msgType; + pMsgSendInfo->fp = loadSysTableCallback; + pMsgSendInfo->requestId = pTaskInfo->id.queryId; + + int64_t transporterId = 0; + int32_t code = + asyncSendMsgToServer(pInfo->readHandle.pMsgCb->clientRpc, &pInfo->epSet, &transporterId, pMsgSendInfo); + tsem_wait(&pInfo->ready); + + if (pTaskInfo->code) { + qDebug("%s load meta data from mnode failed, totalRows:%" PRIu64 ", code:%s", GET_TASKID(pTaskInfo), + pInfo->loadInfo.totalRows, tstrerror(pTaskInfo->code)); + return NULL; + } + + SRetrieveMetaTableRsp* pRsp = pInfo->pRsp; + pInfo->req.showId = pRsp->handle; + + if (pRsp->numOfRows == 0 || pRsp->completed) { + pOperator->status = OP_EXEC_DONE; + qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64, GET_TASKID(pTaskInfo), + pRsp->numOfRows, pInfo->loadInfo.totalRows); + + if (pRsp->numOfRows == 0) { + taosMemoryFree(pRsp); + return NULL; + } + } + + char* pStart = pRsp->data; + extractDataBlockFromFetchRsp(pInfo->pRes, pRsp->data, pInfo->matchInfo.pList, &pStart); + updateLoadRemoteInfo(&pInfo->loadInfo, pRsp->numOfRows, pRsp->compLen, startTs, pOperator); + + // todo log the filter info + doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); + taosMemoryFree(pRsp); + if (pInfo->pRes->info.rows > 0) { + return pInfo->pRes; + } else if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + } + } +} + +SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, + const char* pUser, SExecTaskInfo* pTaskInfo) { + SSysTableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SSysTableScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + SScanPhysiNode* pScanNode = &pScanPhyNode->scan; + SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc; + + int32_t num = 0; + int32_t code = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &num, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + pInfo->accountId = pScanPhyNode->accountId; + pInfo->pUser = taosMemoryStrDup((void*)pUser); + pInfo->sysInfo = pScanPhyNode->sysInfo; + pInfo->showRewrite = pScanPhyNode->showRewrite; + pInfo->pRes = createResDataBlock(pDescNode); + + pInfo->pCondition = pScanNode->node.pConditions; + code = filterInitFromNode(pScanNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + initResultSizeInfo(&pOperator->resultInfo, 4096); + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); + + tNameAssign(&pInfo->name, &pScanNode->tableName); + const char* name = tNameGetTableName(&pInfo->name); + + if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || + strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { + pInfo->readHandle = *(SReadHandle*)readHandle; + } else { + tsem_init(&pInfo->ready, 0, 0); + pInfo->epSet = pScanPhyNode->mgmtEpSet; + pInfo->readHandle = *(SReadHandle*)readHandle; + } + + setOperatorInfo(pOperator, "SysTableScanOperator", QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, false, OP_NOT_OPENED, + pInfo, pTaskInfo); + pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, NULL); + return pOperator; + + _error: + if (pInfo != NULL) { + destroySysScanOperator(pInfo); + } + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} + +void destroySysScanOperator(void* param) { + SSysTableScanInfo* pInfo = (SSysTableScanInfo*)param; + tsem_destroy(&pInfo->ready); + blockDataDestroy(pInfo->pRes); + + const char* name = tNameGetTableName(&pInfo->name); + if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || + strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) { + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + } + if (pInfo->pIdx) { + taosArrayDestroy(pInfo->pIdx->uids); + taosMemoryFree(pInfo->pIdx); + pInfo->pIdx = NULL; + } + + taosArrayDestroy(pInfo->matchInfo.pList); + taosMemoryFreeClear(pInfo->pUser); + + taosMemoryFreeClear(param); +} + +int32_t loadSysTableCallback(void* param, SDataBuf* pMsg, int32_t code) { + SOperatorInfo* operator=(SOperatorInfo*) param; + SSysTableScanInfo* pScanResInfo = (SSysTableScanInfo*)operator->info; + if (TSDB_CODE_SUCCESS == code) { + pScanResInfo->pRsp = pMsg->pData; + + SRetrieveMetaTableRsp* pRsp = pScanResInfo->pRsp; + pRsp->numOfRows = htonl(pRsp->numOfRows); + pRsp->useconds = htobe64(pRsp->useconds); + pRsp->handle = htobe64(pRsp->handle); + pRsp->compLen = htonl(pRsp->compLen); + } else { + operator->pTaskInfo->code = code; + } + + tsem_post(&pScanResInfo->ready); + return TSDB_CODE_SUCCESS; +} + +SSDataBlock* doFilterResult(SSDataBlock* pDataBlock, SFilterInfo* pFilterInfo) { + if (pFilterInfo == NULL) { + return pDataBlock->info.rows == 0 ? NULL : pDataBlock; + } + + doFilter(pDataBlock, pFilterInfo, NULL); + return pDataBlock->info.rows == 0 ? NULL : pDataBlock; +} + +static int32_t sysChkFilter__Comm(SNode* pNode) { + // impl + SOperatorNode* pOper = (SOperatorNode*)pNode; + EOperatorType opType = pOper->opType; + if (opType != OP_TYPE_EQUAL && opType != OP_TYPE_LOWER_EQUAL && opType != OP_TYPE_LOWER_THAN && + opType != OP_TYPE_GREATER_EQUAL && opType != OP_TYPE_GREATER_THAN) { + return -1; + } + return 0; +} + +static int32_t sysChkFilter__DBName(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + + if (pOper->opType != OP_TYPE_EQUAL && pOper->opType != OP_TYPE_NOT_EQUAL) { + return -1; + } + + SValueNode* pVal = (SValueNode*)pOper->pRight; + if (!IS_STR_DATA_TYPE(pVal->node.resType.type)) { + return -1; + } + + return 0; +} +static int32_t sysChkFilter__VgroupId(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { + return -1; + } + return sysChkFilter__Comm(pNode); +} +static int32_t sysChkFilter__TableName(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + if (!IS_STR_DATA_TYPE(pVal->node.resType.type)) { + return -1; + } + return sysChkFilter__Comm(pNode); +} +static int32_t sysChkFilter__CreateTime(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + + if (!IS_TIMESTAMP_TYPE(pVal->node.resType.type)) { + return -1; + } + return sysChkFilter__Comm(pNode); +} + +static int32_t sysChkFilter__Ncolumn(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + + if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { + return -1; + } + return sysChkFilter__Comm(pNode); +} +static int32_t sysChkFilter__Ttl(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + + if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { + return -1; + } + return sysChkFilter__Comm(pNode); +} +static int32_t sysChkFilter__STableName(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + if (!IS_STR_DATA_TYPE(pVal->node.resType.type)) { + return -1; + } + return sysChkFilter__Comm(pNode); +} +static int32_t sysChkFilter__Uid(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { + return -1; + } + return sysChkFilter__Comm(pNode); +} +static int32_t sysChkFilter__Type(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { + return -1; + } + return sysChkFilter__Comm(pNode); +} +static int32_t optSysTabFilteImpl(void* arg, SNode* cond, SArray* result) { + if (optSysCheckOper(cond) != 0) return -1; + + SOperatorNode* pNode = (SOperatorNode*)cond; + + int8_t i = 0; + for (; i < SYSTAB_FILTER_DICT_SIZE; i++) { + if (strcmp(filterDict[i].name, ((SColumnNode*)(pNode->pLeft))->colName) == 0) { + break; + } + } + if (i >= SYSTAB_FILTER_DICT_SIZE) return -1; + + if (filterDict[i].chkFunc(cond) != 0) return -1; + + return filterDict[i].fltFunc(arg, cond, result); +} + +static int32_t optSysCheckOper(SNode* pOpear) { + if (nodeType(pOpear) != QUERY_NODE_OPERATOR) return -1; + + SOperatorNode* pOper = (SOperatorNode*)pOpear; + if (pOper->opType < OP_TYPE_GREATER_THAN || pOper->opType > OP_TYPE_NOT_EQUAL) { + return -1; + } + + if (nodeType(pOper->pLeft) != QUERY_NODE_COLUMN || nodeType(pOper->pRight) != QUERY_NODE_VALUE) { + return -1; + } + return 0; +} + +static FORCE_INLINE int optSysBinarySearch(SArray* arr, int s, int e, uint64_t k) { + uint64_t v; + int32_t m; + while (s <= e) { + m = s + (e - s) / 2; + v = *(uint64_t*)taosArrayGet(arr, m); + if (v >= k) { + e = m - 1; + } else { + s = m + 1; + } + } + return s; +} + +void optSysIntersection(SArray* in, SArray* out) { + int32_t sz = (int32_t)taosArrayGetSize(in); + if (sz <= 0) { + return; + } + MergeIndex* mi = taosMemoryCalloc(sz, sizeof(MergeIndex)); + for (int i = 0; i < sz; i++) { + SArray* t = taosArrayGetP(in, i); + mi[i].len = (int32_t)taosArrayGetSize(t); + mi[i].idx = 0; + } + + SArray* base = taosArrayGetP(in, 0); + for (int i = 0; i < taosArrayGetSize(base); i++) { + uint64_t tgt = *(uint64_t*)taosArrayGet(base, i); + bool has = true; + for (int j = 1; j < taosArrayGetSize(in); j++) { + SArray* oth = taosArrayGetP(in, j); + int mid = optSysBinarySearch(oth, mi[j].idx, mi[j].len - 1, tgt); + if (mid >= 0 && mid < mi[j].len) { + uint64_t val = *(uint64_t*)taosArrayGet(oth, mid); + has = (val == tgt ? true : false); + mi[j].idx = mid; + } else { + has = false; + } + } + if (has == true) { + taosArrayPush(out, &tgt); + } + } + taosMemoryFreeClear(mi); +} + +static int tableUidCompare(const void* a, const void* b) { + int64_t u1 = *(int64_t*)a; + int64_t u2 = *(int64_t*)b; + if (u1 == u2) { + return 0; + } + return u1 < u2 ? -1 : 1; +} + +static int32_t optSysMergeRslt(SArray* mRslt, SArray* rslt) { + // TODO, find comm mem from mRslt + for (int i = 0; i < taosArrayGetSize(mRslt); i++) { + SArray* arslt = taosArrayGetP(mRslt, i); + taosArraySort(arslt, tableUidCompare); + } + optSysIntersection(mRslt, rslt); + return 0; +} + +static int32_t optSysSpecialColumn(SNode* cond) { + SOperatorNode* pOper = (SOperatorNode*)cond; + SColumnNode* pCol = (SColumnNode*)pOper->pLeft; + for (int i = 0; i < sizeof(SYSTABLE_SPECIAL_COL) / sizeof(SYSTABLE_SPECIAL_COL[0]); i++) { + if (0 == strcmp(pCol->colName, SYSTABLE_SPECIAL_COL[i])) { + return 1; + } + } + return 0; +} + +static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result) { + int ret = -1; + if (nodeType(cond) == QUERY_NODE_OPERATOR) { + ret = optSysTabFilteImpl(arg, cond, result); + if (ret == 0) { + SOperatorNode* pOper = (SOperatorNode*)cond; + SColumnNode* pCol = (SColumnNode*)pOper->pLeft; + if (0 == strcmp(pCol->colName, "create_time")) { + return 0; + } + return -1; + } + return ret; + } + + if (nodeType(cond) != QUERY_NODE_LOGIC_CONDITION || ((SLogicConditionNode*)cond)->condType != LOGIC_COND_TYPE_AND) { + return ret; + } + + SLogicConditionNode* pNode = (SLogicConditionNode*)cond; + SNodeList* pList = (SNodeList*)pNode->pParameterList; + + int32_t len = LIST_LENGTH(pList); + + bool hasIdx = false; + bool hasRslt = true; + SArray* mRslt = taosArrayInit(len, POINTER_BYTES); + + SListCell* cell = pList->pHead; + for (int i = 0; i < len; i++) { + if (cell == NULL) break; + + SArray* aRslt = taosArrayInit(16, sizeof(int64_t)); + + ret = optSysTabFilteImpl(arg, cell->pNode, aRslt); + if (ret == 0) { + // has index + hasIdx = true; + if (optSysSpecialColumn(cell->pNode) == 0) { + taosArrayPush(mRslt, &aRslt); + } else { + // db_name/vgroup not result + taosArrayDestroy(aRslt); + } + } else if (ret == -2) { + // current vg + hasIdx = true; + hasRslt = false; + taosArrayDestroy(aRslt); + break; + } else { + taosArrayDestroy(aRslt); + } + cell = cell->pNext; + } + if (hasRslt && hasIdx) { + optSysMergeRslt(mRslt, result); + } + + for (int i = 0; i < taosArrayGetSize(mRslt); i++) { + SArray* aRslt = taosArrayGetP(mRslt, i); + taosArrayDestroy(aRslt); + } + taosArrayDestroy(mRslt); + if (hasRslt == false) { + return -2; + } + if (hasRslt && hasIdx) { + cell = pList->pHead; + for (int i = 0; i < len; i++) { + if (cell == NULL) break; + SOperatorNode* pOper = (SOperatorNode*)cell->pNode; + SColumnNode* pCol = (SColumnNode*)pOper->pLeft; + if (0 == strcmp(pCol->colName, "create_time")) { + return 0; + } + cell = cell->pNext; + } + return -1; + } + return -1; +} \ No newline at end of file -- GitLab