diff --git a/include/common/ttypes.h b/include/common/ttypes.h index eace9d7dd6d96d009eb48e3e52aaf69119094efc..9ff315fc2e97f079f3b7dcbd31d36a69d21bae41 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -340,7 +340,7 @@ typedef struct tDataTypeDescriptor { } tDataTypeDescriptor; extern tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX]; -bool isValidDataType(int32_t type); +bool isValidDataType(int32_t type); void assignVal(char *val, const char *src, int32_t len, int32_t type); void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 01497cc05945618fc77f1112c623b3a8111ab5bd..92a8924e734fb0a6cb03d59ad37feed2b5831824 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -539,6 +539,12 @@ typedef struct { SSnapContext* sContext; } SStreamRawScanInfo; +typedef struct SSysTableIndex { + int8_t init; + SArray *uids; + int32_t lastIdx; +} SSysTableIndex; + typedef struct SSysTableScanInfo { SRetrieveMetaTableRsp* pRsp; SRetrieveTableReq req; @@ -551,6 +557,7 @@ typedef struct SSysTableScanInfo { bool showRewrite; SNode* pCondition; // db_name filter condition, to discard data that are not in current database SMTbCursor* pCur; // cursor for iterate the local table meta store. + SSysTableIndex* pIdx; // idx for local table meta SArray* scanCols; // SArray scan column id list SName name; SSDataBlock* pRes; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 5c1b3f0f609553cdaaf39b3f8397f9c41a451316..203b721c30e6e9b29ec2865351683abca0ebdfaf 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -56,6 +56,7 @@ typedef struct { void* pVnode; } SSTabFltArg; +static int32_t sysChkFilter__Comm(SNode* pNode); static int32_t sysChkFilter__DBName(SNode* pNode); static int32_t sysChkFilter__VgroupId(SNode* pNode); static int32_t sysChkFilter__TableName(SNode* pNode); @@ -2375,6 +2376,11 @@ static void destroySysScanOperator(void* param) { metaCloseTbCursor(pInfo->pCur); pInfo->pCur = NULL; } + if (pInfo->pIdx) { + taosArrayDestroy(pInfo->pIdx->uids); + taosMemoryFree(pInfo->pIdx); + pInfo->pIdx = NULL; + } taosArrayDestroy(pInfo->scanCols); taosMemoryFreeClear(pInfo->pUser); @@ -2893,6 +2899,10 @@ static int optSysFilterFuncImpl__Equal(void* a, void* b, int16_t dtype) { return optSysDoCompare(func, OP_TYPE_EQUAL, a, b); } +static int optSysFilterFuncImpl__NoEqual(void* a, void* b, int16_t dtype) { + __compar_fn_t func = getComparFunc(dtype, 0); + return optSysDoCompare(func, OP_TYPE_NOT_EQUAL, a, b); +} static __optSysFilter optSysGetFilterFunc(int32_t ctype, bool* reverse) { if (ctype == OP_TYPE_LOWER_EQUAL || ctype == OP_TYPE_LOWER_THAN) { *reverse = true; @@ -2907,6 +2917,8 @@ static __optSysFilter optSysGetFilterFunc(int32_t ctype, bool* reverse) { return optSysFilterFuncImpl__GreaterEqual; else if (ctype == OP_TYPE_EQUAL) return optSysFilterFuncImpl__Equal; + else if (ctype == OP_TYPE_NOT_EQUAL) + return optSysFilterFuncImpl__NoEqual; return NULL; } static int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result) { @@ -2933,7 +2945,7 @@ static int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result) { int ret = func(dbname, pVal->datum.p, TSDB_DATA_TYPE_VARCHAR); if (ret == 0) return 0; - return -1; + return -2; } static int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result) { void* pVnode = ((SSTabFltArg*)arg)->pVnode; @@ -3051,6 +3063,17 @@ static int32_t sysFilte__Type(void* arg, SNode* pNode, SArray* result) { if (func == NULL) return -1; return 0; } +static int32_t sysChkFilter__Comm(SNode* pNode) { + // impl + SOperatorNode* pOper = (SOperatorNode*)pNode; + EOperatorType opType = pOper->opType; + if (opType != OP_TYPE_EQUAL && opType != OP_TYPE_LOWER_EQUAL && opType != OP_TYPE_LOWER_THAN && + OP_TYPE_GREATER_EQUAL && opType != OP_TYPE_GREATER_THAN) { + return -1; + } + return 0; +} + static int32_t sysChkFilter__DBName(SNode* pNode) { SOperatorNode* pOper = (SOperatorNode*)pNode; @@ -3059,7 +3082,7 @@ static int32_t sysChkFilter__DBName(SNode* pNode) { } SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_STR_DATA_TYPE(pVal->typeData)) { + if (!IS_STR_DATA_TYPE(pVal->node.resType.type)) { return -1; } @@ -3068,71 +3091,70 @@ static int32_t sysChkFilter__DBName(SNode* pNode) { static int32_t sysChkFilter__VgroupId(SNode* pNode) { SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_VALID_INT(pVal->typeData)) { + if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { return -1; } - return 0; + return sysChkFilter__Comm(pNode); } static int32_t sysChkFilter__TableName(SNode* pNode) { SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_STR_DATA_TYPE(pVal->typeData)) { + if (!IS_STR_DATA_TYPE(pVal->node.resType.type)) { return -1; } - return 0; + return sysChkFilter__Comm(pNode); } static int32_t sysChkFilter__CreateTime(SNode* pNode) { SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_VALID_BIGINT(pVal->typeData)) { + if (!IS_TIMESTAMP_TYPE(pVal->node.resType.type)) { return -1; } - - return 0; + return sysChkFilter__Comm(pNode); } static int32_t sysChkFilter__Ncolumn(SNode* pNode) { SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_VALID_INT(pVal->typeData)) { + if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { return -1; } - return 0; + return sysChkFilter__Comm(pNode); } static int32_t sysChkFilter__Ttl(SNode* pNode) { SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_VALID_BIGINT(pVal->typeData)) { + if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { return -1; } - return 0; + return sysChkFilter__Comm(pNode); } static int32_t sysChkFilter__STableName(SNode* pNode) { SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_STR_DATA_TYPE(pVal->typeData)) { + if (!IS_STR_DATA_TYPE(pVal->node.resType.type)) { return -1; } - return 0; + return sysChkFilter__Comm(pNode); } static int32_t sysChkFilter__Uid(SNode* pNode) { SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_VALID_BIGINT(pVal->typeData)) { + if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { return -1; } - return 0; + return sysChkFilter__Comm(pNode); } static int32_t sysChkFilter__Type(SNode* pNode) { SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_VALID_INT(pVal->typeData)) { + if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { return -1; } - return 0; + return sysChkFilter__Comm(pNode); } static int32_t optSysTabFilteImpl(void* arg, SNode* cond, SArray* result) { if (optSysCheckOper(cond) != 0) return -1; @@ -3191,7 +3213,7 @@ static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result) { if (len <= 0) return ret; bool hasIdx = false; - int hasRslt = true; + bool hasRslt = true; SArray* mRslt = taosArrayInit(len, POINTER_BYTES); SListCell* cell = pList->pHead; @@ -3221,15 +3243,22 @@ static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result) { for (int i = 0; i < taosArrayGetSize(mRslt); i++) { SArray* aRslt = taosArrayGetP(mRslt, i); taosArrayDestroy(aRslt); - }; + } taosArrayDestroy(mRslt); - - return hasIdx == true ? 0 : -1; + if (hasRslt == false) { + return -2; + } + if (hasRslt && hasIdx) { + return 0; + } + return -1; } -static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator, SArray* pUids) { +static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SSysTableScanInfo* pInfo = pOperator->info; + + SSysTableIndex* pIdx = pInfo->pIdx; blockDataCleanup(pInfo->pRes); int32_t numOfRows = 0; @@ -3249,10 +3278,10 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator, SArr SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TABLES); blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); - char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - - for (int i = 0; i < taosArrayGetSize(pUids); i++) { - tb_uid_t* uid = taosArrayGet(pUids, i); + char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + int32_t i = pIdx->lastIdx; + for (; i < taosArrayGetSize(pIdx->uids); i++) { + tb_uid_t* uid = taosArrayGet(pIdx->uids, i); SMetaReader mr = {0}; metaReaderInit(&mr, pInfo->readHandle.meta, 0); @@ -3398,12 +3427,13 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator, SArr numOfRows = 0; } - blockDataDestroy(p); + if (i >= taosArrayGetSize(pIdx->uids)) { + doSetOperatorCompleted(pOperator); + } else { + pIdx->lastIdx = i; + } - // todo temporarily free the cursor here, the true reason why the free is not valid needs to be found - // if (ret != 0) { - doSetOperatorCompleted(pOperator); - //} + blockDataDestroy(p); pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; @@ -3604,25 +3634,32 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { doSetOperatorCompleted(pOperator); return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; } else { - if (pCondition != NULL) { + if (pCondition != NULL && pInfo->pIdx == NULL) { SSTabFltArg arg = {.pMeta = pInfo->readHandle.meta, .pVnode = pInfo->readHandle.vnode}; - SArray* uids = taosArrayInit(128, sizeof(int64_t)); + + SSysTableIndex* idx = taosMemoryMalloc(sizeof(SSysTableIndex)); + idx->init = 0; + idx->uids = taosArrayInit(128, sizeof(int64_t)); + idx->lastIdx = 0; + pInfo->pIdx = idx; + + SArray* uids = idx->uids; int flt = optSysTabFilte(&arg, pCondition, uids); if (flt == 0) { - SSDataBlock* blk = sysTableBuildUserTablesByUids(pOperator, uids); - taosArrayDestroy(uids); + SSDataBlock* blk = sysTableBuildUserTablesByUids(pOperator); return blk; - // filter succ, build result by uids } else if (flt == -2) { - // filter succe and not result, build empty result - taosArrayDestroy(uids); + qDebug("%s failed to get sys table info by idx, empty result", GET_TASKID(pTaskInfo)); return NULL; } else if (flt == -1) { - // iterate meta and filte one by one + qDebug("%s failed to get sys table info by idx, scan sys table one by one", GET_TASKID(pTaskInfo)); } - taosArrayDestroy(uids); + } else if (pCondition != NULL && pInfo->pIdx != NULL) { + SSDataBlock* blk = sysTableBuildUserTablesByUids(pOperator); + return blk; } + return sysTableBuildUserTables(pOperator); } return NULL;