diff --git a/include/common/systable.h b/include/common/systable.h index 8b0bb4a3fba107e1d74bee6885c39ae06d425a19..d2c28941c77ae92c7a52c6e9df89de1650ad4360 100644 --- a/include/common/systable.h +++ b/include/common/systable.h @@ -52,6 +52,7 @@ extern "C" { #define TSDB_PERFS_TABLE_OFFSETS "offsets" #define TSDB_PERFS_TABLE_TRANS "trans" #define TSDB_PERFS_TABLE_STREAMS "streams" +#define TSDB_PERFS_TABLE_APPS "apps" typedef struct SSysDbTableSchema { const char* name; diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 159c120dde65428411c2f938d1430b4466519d95..8dabd56934677917e11f0ccf2c97c7490087f7d7 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -261,55 +261,49 @@ int32_t ctgInitGetTbIndexTask(SCtgJob *pJob, int32_t taskIdx, SName *name) { } -int32_t ctgHandleForceUpdate(SCatalog* pCtg, SCtgJob *pJob, const SCatalogReq* pReq) { - int32_t dbNum = pJob->dbCfgNum + pJob->dbVgNum + pJob->dbInfoNum; - if (dbNum > 0) { - if (dbNum > pJob->dbCfgNum && dbNum > pJob->dbVgNum && dbNum > pJob->dbInfoNum) { - SHashObj* pDb = taosHashInit(dbNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (NULL == pDb) { - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); - } - - for (int32_t i = 0; i < pJob->dbVgNum; ++i) { - char* dbFName = taosArrayGet(pReq->pDbVgroup, i); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); - } +int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob *pJob, const SCatalogReq* pReq) { + SHashObj* pDb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (NULL == pDb) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + for (int32_t i = 0; i < pJob->dbVgNum; ++i) { + char* dbFName = taosArrayGet(pReq->pDbVgroup, i); + taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + } - for (int32_t i = 0; i < pJob->dbCfgNum; ++i) { - char* dbFName = taosArrayGet(pReq->pDbCfg, i); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); - } + for (int32_t i = 0; i < pJob->dbCfgNum; ++i) { + char* dbFName = taosArrayGet(pReq->pDbCfg, i); + taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + } - for (int32_t i = 0; i < pJob->dbInfoNum; ++i) { - char* dbFName = taosArrayGet(pReq->pDbInfo, i); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); - } + for (int32_t i = 0; i < pJob->dbInfoNum; ++i) { + char* dbFName = taosArrayGet(pReq->pDbInfo, i); + taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + } - char* dbFName = taosHashIterate(pDb, NULL); - while (dbFName) { - ctgDropDbVgroupEnqueue(pCtg, dbFName, true); - dbFName = taosHashIterate(pDb, dbFName); - } + for (int32_t i = 0; i < pJob->tbMetaNum; ++i) { + SName* name = taosArrayGet(pReq->pTableMeta, i); + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(name, dbFName); + taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + } + + for (int32_t i = 0; i < pJob->tbHashNum; ++i) { + SName* name = taosArrayGet(pReq->pTableHash, i); + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(name, dbFName); + taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + } - taosHashCleanup(pDb); - } else { - for (int32_t i = 0; i < pJob->dbVgNum; ++i) { - char* dbFName = taosArrayGet(pReq->pDbVgroup, i); - CTG_ERR_RET(ctgDropDbVgroupEnqueue(pCtg, dbFName, true)); - } - - for (int32_t i = 0; i < pJob->dbCfgNum; ++i) { - char* dbFName = taosArrayGet(pReq->pDbCfg, i); - CTG_ERR_RET(ctgDropDbVgroupEnqueue(pCtg, dbFName, true)); - } - - for (int32_t i = 0; i < pJob->dbInfoNum; ++i) { - char* dbFName = taosArrayGet(pReq->pDbInfo, i); - CTG_ERR_RET(ctgDropDbVgroupEnqueue(pCtg, dbFName, true)); - } - } + char* dbFName = taosHashIterate(pDb, NULL); + while (dbFName) { + ctgDropDbVgroupEnqueue(pCtg, dbFName, true); + dbFName = taosHashIterate(pDb, dbFName); } + taosHashCleanup(pDb); + int32_t tbNum = pJob->tbMetaNum + pJob->tbHashNum; if (tbNum > 0) { if (tbNum > pJob->tbMetaNum && tbNum > pJob->tbHashNum) { @@ -404,7 +398,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, uint6 } if (pReq->forceUpdate) { - CTG_ERR_JRET(ctgHandleForceUpdate(pCtg, pJob, pReq)); + CTG_ERR_JRET(ctgHandleForceUpdate(pCtg, *taskNum, pJob, pReq)); } int32_t taskIdx = 0; @@ -790,8 +784,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * _return: if (dbCache) { - ctgRUnlockVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); } ctgHandleTaskEnd(pTask, code); @@ -870,7 +863,7 @@ _return: int32_t ctgHandleGetTbIndexRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; - CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); STableIndex* pOut = (STableIndex*)pTask->msgCtx.out; SArray* pInfo = NULL; @@ -949,7 +942,6 @@ _return: int32_t ctgHandleGetUserRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; - SCtgDBCache *dbCache = NULL; SCtgUserCtx* ctx = (SCtgUserCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; bool pass = false; @@ -1018,7 +1010,7 @@ int32_t ctgAsyncRefreshTbMeta(SCtgTask *pTask) { CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (dbCache) { SVgroupInfo vgInfo = {0}; - CTG_ERR_RET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, ctx->pName, &vgInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, ctx->pName, &vgInfo)); ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); @@ -1067,6 +1059,9 @@ int32_t ctgLaunchGetDbVgTask(SCtgTask *pTask) { CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pCtx->dbFName, &dbCache)); if (NULL != dbCache) { CTG_ERR_JRET(ctgGenerateVgList(pCtg, dbCache->vgCache.vgInfo->vgHash, (SArray**)&pTask->res)); + + ctgReleaseVgInfoToCache(pCtg, dbCache); + dbCache = NULL; CTG_ERR_JRET(ctgHandleTaskEnd(pTask, 0)); } else { @@ -1101,6 +1096,9 @@ int32_t ctgLaunchGetTbHashTask(SCtgTask *pTask) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, pCtx->pName, (SVgroupInfo*)pTask->res)); + + ctgReleaseVgInfoToCache(pCtg, dbCache); + dbCache = NULL; CTG_ERR_JRET(ctgHandleTaskEnd(pTask, 0)); } else { @@ -1176,6 +1174,9 @@ int32_t ctgLaunchGetDbInfoTask(SCtgTask *pTask) { pInfo->vgVer = dbCache->vgCache.vgInfo->vgVersion; pInfo->dbId = dbCache->dbId; pInfo->tbNum = dbCache->vgCache.vgInfo->numOfTable; + + ctgReleaseVgInfoToCache(pCtg, dbCache); + dbCache = NULL; } else { pInfo->vgVer = CTG_DEFAULT_INVALID_VERSION; } diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index 9de1ea22be82fbde1c689c30ce84a5940eaed4fe..ff93bedb213607c3592b057c12beb2ed2f733d54 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -19,7 +19,7 @@ #include "catalogInt.h" extern SCatalogMgmt gCtgMgmt; -SCtgDebug gCTGDebug = {0}; +SCtgDebug gCTGDebug = {.lockEnable = true, .apiEnable = true}; void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) { ASSERT(*(int32_t*)param == 1); diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 1cb2b99a67e82f4eb95d72daa392336164abfc50..e97c34dc268b61b402304aeb51689a99926484ec 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -675,7 +675,8 @@ int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes) { for (int32_t i = 0; i < num; ++i) { STableIndexInfo *pInfo = taosArrayGet(pIndex, i); - taosArrayPush(*pRes, pInfo); + pInfo = taosArrayPush(*pRes, pInfo); + pInfo->expr = strdup(pInfo->expr); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 701c7ee0f3a0faa44bb4f4f816bdafa11f972be4..2b6736e44b4cdb7013eba6f2f5c4bbe722e8682a 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -391,7 +391,9 @@ typedef struct SStreamBlockScanInfo { void* streamBlockReader;// stream block reader handle SArray* pColMatchInfo; // SNode* pCondition; + int32_t tsArrayIndex; SArray* tsArray; + uint64_t groupId; SUpdateInfo* pUpdateInfo; SExprInfo* pPseudoExpr; @@ -582,6 +584,7 @@ typedef struct SPartitionOperatorInfo { int32_t* columnOffset; // start position for each column data void* pGroupIter; // group iterator int32_t pageIndex; // page index of current group + SSDataBlock* pUpdateRes; } SPartitionOperatorInfo; typedef struct SWindowRowsSup { @@ -907,6 +910,7 @@ int32_t compareTimeWindow(const void* p1, const void* p2, const void* param); int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, const int32_t* rowCellOffset, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); + int32_t getTableList(void* metaHandle, int32_t tableType, uint64_t tableUid, STableListInfo* pListInfo, SNode* pTagCond); int32_t createMultipleDataReaders(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, @@ -914,6 +918,9 @@ int32_t createMultipleDataReaders(STableScanPhysiNode* pTableScanNode, SReadHand uint64_t taskId, SNode* pTagCond); SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SArray* dataReaders, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo); + +void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex); + #ifdef __cplusplus } #endif diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ac4b4d45b6b9cf704870bd4575aca1ff481207e6..a5ba48f871c0c328a72b1074b2632450d06c8bc6 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -751,60 +751,127 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) { return true; } -static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo) { - SSDataBlock* pResult = NULL; - pResult = doTableScan(pInfo->pOperatorDumy); - if (pResult == NULL) { - if (prepareDataScan(pInfo)) { - // scan next window data - pResult = doTableScan(pInfo->pOperatorDumy); +static void copyOneRow(SSDataBlock* dest, SSDataBlock* source, int32_t sourceRowId) { + for (int32_t j = 0; j < source->info.numOfCols; j++) { + SColumnInfoData* pDestCol = (SColumnInfoData*)taosArrayGet(dest->pDataBlock, j); + SColumnInfoData* pSourceCol = (SColumnInfoData*)taosArrayGet(source->pDataBlock, j); + if (colDataIsNull_s(pSourceCol, sourceRowId)) { + colDataAppendNULL(pDestCol, dest->info.rows); + } else { + colDataAppend(pDestCol, dest->info.rows, colDataGetData(pSourceCol, sourceRowId), false); } } - return pResult; + dest->info.rows++; } -static void getUpdateDataBlock(SStreamBlockScanInfo* pInfo, bool invertible, SSDataBlock* pBlock, - SSDataBlock* pUpdateBlock) { - SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); - ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP); - TSKEY* ts = (TSKEY*)pColDataInfo->pData; - for (int32_t i = 0; i < pBlock->info.rows; i++) { - if (updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, ts[i])) { - taosArrayPush(pInfo->tsArray, ts + i); +static uint64_t getGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t rowId) { + uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t)); + if (groupId) { + return *groupId; + } + return 0; + /* Todo(liuyao) for partition by column + recordNewGroupKeys(pTableScanInfo->pGroupCols, pTableScanInfo->pGroupColVals, pBlock, rowId); + int32_t len = buildGroupKeys(pTableScanInfo->keyBuf, pTableScanInfo->pGroupColVals); + uint64_t resId = 0; + uint64_t* groupId = taosHashGet(pTableScanInfo->pGroupSet, pTableScanInfo->keyBuf, len); + if (groupId) { + return *groupId; + } else if (len != 0) { + resId = calcGroupId(pTableScanInfo->keyBuf, len); + taosHashPut(pTableScanInfo->pGroupSet, pTableScanInfo->keyBuf, len, &resId, sizeof(uint64_t)); + } + return resId; + */ +} + +static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo) { + while (1) { + SSDataBlock* pResult = NULL; + pResult = doTableScan(pInfo->pOperatorDumy); + if (pResult == NULL) { + if (prepareDataScan(pInfo)) { + // scan next window data + pResult = doTableScan(pInfo->pOperatorDumy); + } + } + if (!pResult) { + return NULL; + } + + if (pResult->info.groupId == pInfo->groupId) { + return pResult; } } - if (!pUpdateBlock) { - taosArrayClear(pInfo->tsArray); - return; + +/* Todo(liuyao) for partition by column + SSDataBlock* pBlock = createOneDataBlock(pResult, true); + blockDataCleanup(pResult); + for (int32_t i = 0; i < pBlock->info.rows; i++) { + uint64_t id = getGroupId(pInfo->pOperatorDumy, pBlock, i); + if (id == pInfo->groupId) { + copyOneRow(pResult, pBlock, i); + } } + return pResult; +*/ +} + +static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDataBlock* pUpdateBlock) { + blockDataCleanup(pUpdateBlock); int32_t size = taosArrayGetSize(pInfo->tsArray); - if (size > 0 && invertible) { - // Todo(liuyao) get from tsdb - // SSDataBlock* p = createOneDataBlock(pBlock, true); - // p->info.type = STREAM_INVERT; - // taosArrayClear(pInfo->tsArray); - // return p; + if (pInfo->tsArrayIndex < size) { SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pUpdateBlock->pDataBlock, pInfo->primaryTsIndex); ASSERT(pCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); blockDataEnsureCapacity(pUpdateBlock, size); - for (int32_t i = 0; i < size; i++) { - TSKEY* pTs = (TSKEY*)taosArrayGet(pInfo->tsArray, i); - colDataAppend(pCol, i, (char*)pTs, false); - } - for (int32_t i = 0; i < pUpdateBlock->info.numOfCols; i++) { - if (i == pInfo->primaryTsIndex) { - continue; + ASSERT(pBlock->info.numOfCols == pUpdateBlock->info.numOfCols); + + int32_t rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, pInfo->tsArrayIndex); + pInfo->groupId = getGroupId(pInfo->pOperatorDumy, pBlock, rowId); + int32_t i = 0; + for ( ; i < size; i++) { + rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, i + pInfo->tsArrayIndex); + uint64_t id = getGroupId(pInfo->pOperatorDumy, pBlock, rowId); + if (pInfo->groupId != id) { + break; } - SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pUpdateBlock->pDataBlock, i); - colDataAppendNNULL(pCol, 0, size); + copyOneRow(pUpdateBlock, pBlock, rowId); } - pUpdateBlock->info.rows = size; + pUpdateBlock->info.rows = i; + pInfo->tsArrayIndex += i; + pUpdateBlock->info.groupId = pInfo->groupId; pUpdateBlock->info.type = STREAM_REPROCESS; blockDataUpdateTsWindow(pUpdateBlock, 0); + } + // all rows have same group id + ASSERT(pInfo->tsArrayIndex >= size); + if (size > 0 && pInfo->tsArrayIndex == size) { taosArrayClear(pInfo->tsArray); } } +static void getUpdateDataBlock(SStreamBlockScanInfo* pInfo, bool invertible, SSDataBlock* pBlock, + SSDataBlock* pUpdateBlock) { + SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); + ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP); + TSKEY* ts = (TSKEY*)pColDataInfo->pData; + for (int32_t rowId = 0; rowId < pBlock->info.rows; rowId++) { + if (updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, ts[rowId])) { + taosArrayPush(pInfo->tsArray, &rowId); + } + } + if (!pUpdateBlock) { + taosArrayClear(pInfo->tsArray); + return; + } + setUpdateData(pInfo, pBlock, pUpdateBlock); + // Todo(liuyao) get from tsdb + // SSDataBlock* p = createOneDataBlock(pBlock, true); + // p->info.type = STREAM_INVERT; + // taosArrayClear(pInfo->tsArray); + // return p; +} + static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -834,7 +901,6 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; return pInfo->pRes; } else if (pInfo->scanMode == STREAM_SCAN_FROM_UPDATERES) { - blockDataCleanup(pInfo->pRes); pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER; if (!isStateWindow(pInfo)) { prepareDataScan(pInfo); @@ -849,7 +915,15 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER) { SSDataBlock* pSDB = doDataScan(pInfo); if (pSDB == NULL) { - pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; + setUpdateData(pInfo, pInfo->pRes, pInfo->pUpdateRes); + if (pInfo->pUpdateRes->info.rows > 0) { + if (!isStateWindow(pInfo)) { + prepareDataScan(pInfo); + } + return pInfo->pUpdateRes; + } else { + pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; + } } else { getUpdateDataBlock(pInfo, true, pSDB, NULL); return pSDB; @@ -942,7 +1016,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { if (rows == 0) { pOperator->status = OP_EXEC_DONE; } else if (pInfo->pUpdateInfo) { - blockDataCleanup(pInfo->pUpdateRes); + pInfo->tsArrayIndex = 0; getUpdateDataBlock(pInfo, true, pInfo->pRes, pInfo->pUpdateRes); if (pInfo->pUpdateRes->info.rows > 0) { if (pInfo->pUpdateRes->info.type == STREAM_REPROCESS) { @@ -1021,7 +1095,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan goto _error; } - pInfo->tsArray = taosArrayInit(4, sizeof(TSKEY)); + pInfo->tsArray = taosArrayInit(4, sizeof(int32_t)); if (pInfo->tsArray == NULL) { goto _error; } @@ -1048,6 +1122,8 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan pInfo->pOperatorDumy = pTableScanDummy; pInfo->interval = pSTInfo->interval; pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1}; + pInfo->groupId = 0; + pOperator->name = "StreamBlockScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; pOperator->blocking = false; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 01514740080655f0ba50ec396c3e476538ef0d79..5a18649cab2fbc267afe00e09cf79212f56175e3 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1985,7 +1985,7 @@ static void clearUpdateDataBlock(SSDataBlock* pBlock) { blockDataCleanup(pBlock); } -static void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) { +void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) { ASSERT(pDest->info.capacity >= pSource->info.rows); clearUpdateDataBlock(pDest); SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0); @@ -1997,6 +1997,8 @@ static void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_ colDataAppendNNULL(pCol, 0, pSource->info.rows); } pDest->info.rows = pSource->info.rows; + pDest->info.groupId = pSource->info.groupId; + pDest->info.type = pSource->info.type; blockDataUpdateTsWindow(pDest, 0); } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index c6ab9f859d735a4599a9fbdd6401b1b8fae0516c..94208043e7835bbeb9c4c487c02333c59d80bc45 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2345,14 +2345,7 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); SAPercentileInfo* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); ASSERT(pDBuf->algo == pSBuf->algo); - if (pDBuf->algo == APERCT_ALGO_TDIGEST) { - tdigestMerge(pDBuf->pTDigest, pSBuf->pTDigest); - } else { - SHistogramInfo* pTmp = tHistogramMerge(pDBuf->pHisto, pSBuf->pHisto, MAX_HISTOGRAM_BIN); - memcpy(pDBuf->pHisto, pTmp, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1)); - pDBuf->pHisto->elems = (SHistBin*)((char*)pDBuf->pHisto + sizeof(SHistogramInfo)); - tHistogramDestroy(&pTmp); - } + apercentileTransferInfo(pSBuf, pDBuf); pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 9e0de484208c514874f8029caf9ea9d6a4661ac9..a63e7a5790a1698f6479195ef3ff7b73ae5cfcbf 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4128,12 +4128,14 @@ static const char* getSysDbName(ENodeType type) { case QUERY_NODE_SHOW_SNODES_STMT: case QUERY_NODE_SHOW_LICENCE_STMT: case QUERY_NODE_SHOW_CLUSTER_STMT: + case QUERY_NODE_SHOW_VARIABLE_STMT: return TSDB_INFORMATION_SCHEMA_DB; case QUERY_NODE_SHOW_CONNECTIONS_STMT: case QUERY_NODE_SHOW_QUERIES_STMT: case QUERY_NODE_SHOW_TOPICS_STMT: case QUERY_NODE_SHOW_STREAMS_STMT: case QUERY_NODE_SHOW_TRANSACTIONS_STMT: + case QUERY_NODE_SHOW_APPS_STMT: return TSDB_PERFORMANCE_SCHEMA_DB; default: break; @@ -4183,6 +4185,10 @@ static const char* getSysTableName(ENodeType type) { return TSDB_PERFS_TABLE_TOPICS; case QUERY_NODE_SHOW_TRANSACTIONS_STMT: return TSDB_PERFS_TABLE_TRANS; + case QUERY_NODE_SHOW_VARIABLE_STMT: + return TSDB_INS_TABLE_CONFIGS; + case QUERY_NODE_SHOW_APPS_STMT: + return TSDB_PERFS_TABLE_APPS; default: break; } @@ -5237,6 +5243,8 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { case QUERY_NODE_SHOW_CLUSTER_STMT: case QUERY_NODE_SHOW_TOPICS_STMT: case QUERY_NODE_SHOW_TRANSACTIONS_STMT: + case QUERY_NODE_SHOW_VARIABLE_STMT: + case QUERY_NODE_SHOW_APPS_STMT: code = rewriteShow(pCxt, pQuery); break; case QUERY_NODE_CREATE_TABLE_STMT: diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 65d73b4cdb706252c534b45f90eb8fed50f60507..41b80eaaa8b4486c1974f782fc8c95b8452ddcd5 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -104,10 +104,14 @@ static bool osdMayBeOptimized(SLogicNode* pNode) { return false; } if (NULL == pNode->pParent || (QUERY_NODE_LOGIC_PLAN_WINDOW != nodeType(pNode->pParent) && - QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode->pParent))) { + QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode->pParent) && + QUERY_NODE_LOGIC_PLAN_PARTITION != nodeType(pNode->pParent))) { return false; } - if (QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent)) { + if (QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent) || + (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode->pParent) && + pNode->pParent->pParent && + QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent->pParent)) ) { return true; } return !osdHaveNormalCol(((SAggLogicNode*)pNode->pParent)->pGroupKeys); @@ -217,16 +221,22 @@ static int32_t osdGetDataRequired(SNodeList* pFuncs) { } static void setScanWindowInfo(SScanLogicNode* pScan) { - if (QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pScan->node.pParent)) { - pScan->interval = ((SWindowLogicNode*)pScan->node.pParent)->interval; - pScan->offset = ((SWindowLogicNode*)pScan->node.pParent)->offset; - pScan->sliding = ((SWindowLogicNode*)pScan->node.pParent)->sliding; - pScan->intervalUnit = ((SWindowLogicNode*)pScan->node.pParent)->intervalUnit; - pScan->slidingUnit = ((SWindowLogicNode*)pScan->node.pParent)->slidingUnit; - pScan->triggerType = ((SWindowLogicNode*)pScan->node.pParent)->triggerType; - pScan->watermark = ((SWindowLogicNode*)pScan->node.pParent)->watermark; - pScan->tsColId = ((SColumnNode*)((SWindowLogicNode*)pScan->node.pParent)->pTspk)->colId; - pScan->filesFactor = ((SWindowLogicNode*)pScan->node.pParent)->filesFactor; + SLogicNode* pParent = pScan->node.pParent; + if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pParent) && + pParent->pParent && + QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pParent->pParent)) { + pParent = pParent->pParent; + } + if (QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pParent)) { + pScan->interval = ((SWindowLogicNode*)pParent)->interval; + pScan->offset = ((SWindowLogicNode*)pParent)->offset; + pScan->sliding = ((SWindowLogicNode*)pParent)->sliding; + pScan->intervalUnit = ((SWindowLogicNode*)pParent)->intervalUnit; + pScan->slidingUnit = ((SWindowLogicNode*)pParent)->slidingUnit; + pScan->triggerType = ((SWindowLogicNode*)pParent)->triggerType; + pScan->watermark = ((SWindowLogicNode*)pParent)->watermark; + pScan->tsColId = ((SColumnNode*)((SWindowLogicNode*)pParent)->pTspk)->colId; + pScan->filesFactor = ((SWindowLogicNode*)pParent)->filesFactor; } } diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index 1107e9693e168b72461b35fdc8af827f86fe5970..b6890a85034c27ea2c13402c053518755d315adb 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -243,9 +243,8 @@ class TDTestCase: tdSql.checkRows(2) tdSql.query("select * from jsons1 where jtag->'tag2'!='beijing'") tdSql.checkRows(5) - #open - #tdSql.query("select * from jsons1 where jtag->'tag2'=''") - #tdSql.checkRows(2) + tdSql.query("select * from jsons1 where jtag->'tag2'=''") + tdSql.checkRows(2) # # # where json value is int tdSql.query("select * from jsons1 where jtag->'tag1'=5") @@ -253,11 +252,10 @@ class TDTestCase: tdSql.checkData(0, 1, 2) tdSql.query("select * from jsons1 where jtag->'tag1'=10") tdSql.checkRows(0) - # open - #tdSql.query("select * from jsons1 where jtag->'tag1'<54") - #tdSql.checkRows(3) - #tdSql.query("select * from jsons1 where jtag->'tag1'<=11") - #tdSql.checkRows(3) + tdSql.query("select * from jsons1 where jtag->'tag1'<54") + tdSql.checkRows(4) + tdSql.query("select * from jsons1 where jtag->'tag1'<=11") + tdSql.checkRows(4) tdSql.query("select * from jsons1 where jtag->'tag1'>4") tdSql.checkRows(2) tdSql.query("select * from jsons1 where jtag->'tag1'>=5") @@ -270,31 +268,28 @@ class TDTestCase: # # where json value is double tdSql.query("select * from jsons1 where jtag->'tag1'=1.232") tdSql.checkRows(1) - # open - #tdSql.query("select * from jsons1 where jtag->'tag1'<1.232") - #tdSql.checkRows(0) - #tdSql.query("select * from jsons1 where jtag->'tag1'<=1.232") - #tdSql.checkRows(1) + tdSql.query("select * from jsons1 where jtag->'tag1'<1.232") + tdSql.checkRows(1) + tdSql.query("select * from jsons1 where jtag->'tag1'<=1.232") + tdSql.checkRows(2) tdSql.query("select * from jsons1 where jtag->'tag1'>1.23") tdSql.checkRows(3) tdSql.query("select * from jsons1 where jtag->'tag1'>=1.232") tdSql.checkRows(3) - # open - #tdSql.query("select * from jsons1 where jtag->'tag1'!=1.232") - #tdSql.checkRows(2) + tdSql.query("select * from jsons1 where jtag->'tag1'!=1.232") + tdSql.checkRows(6) tdSql.query("select * from jsons1 where jtag->'tag1'!=3.232") tdSql.checkRows(7) #tdSql.error("select * from jsons1 where jtag->'tag1'/0=3") #tdSql.error("select * from jsons1 where jtag->'tag1'/5=1") # # # where json value is bool - #tdSql.query("select * from jsons1 where jtag->'tag1'=true") - # open - #tdSql.checkRows(0) + tdSql.query("select * from jsons1 where jtag->'tag1'=true") + tdSql.checkRows(0) #tdSql.query("select * from jsons1 where jtag->'tag1'=false") #tdSql.checkRows(1) - #tdSql.query("select * from jsons1 where jtag->'tag1'!=false") - #tdSql.checkRows(0) + tdSql.query("select * from jsons1 where jtag->'tag1'!=false") + tdSql.checkRows(3) #tdSql.error("select * from jsons1 where jtag->'tag1'>false") # # # where json value is null @@ -303,18 +298,17 @@ class TDTestCase: #tdSql.checkRows(1) # # # where json key is null - # open - #tdSql.query("select * from jsons1 where jtag->'tag_no_exist'=3") - #tdSql.checkRows(0) + tdSql.query("select * from jsons1 where jtag->'tag_no_exist'=3") + tdSql.checkRows(0) # # # where json value is not exist - #tdSql.query("select * from jsons1 where jtag->'tag1' is null") - #tdSql.checkData(0, 0, 'jsons1_9') - #tdSql.checkRows(1) - #tdSql.query("select * from jsons1 where jtag->'tag4' is null") - #tdSql.checkRows(9) - #tdSql.query("select * from jsons1 where jtag->'tag3' is not null") - #tdSql.checkRows(4) + tdSql.query("select * from jsons1 where jtag->'tag1' is null") + tdSql.checkData(0, 0, 'jsons1_9') + tdSql.checkRows(2) + tdSql.query("select * from jsons1 where jtag->'tag4' is null") + tdSql.checkRows(9) + tdSql.query("select * from jsons1 where jtag->'tag3' is not null") + tdSql.checkRows(3) # # # test contains tdSql.query("select * from jsons1 where jtag contains 'tag1'") @@ -344,10 +338,10 @@ class TDTestCase: # # # # test with between and - #tdSql.query("select * from jsons1 where jtag->'tag1' between 1 and 30") - #tdSql.checkRows(3) - #tdSql.query("select * from jsons1 where jtag->'tag1' between 'femail' and 'beijing'") - #tdSql.checkRows(2) + tdSql.query("select * from jsons1 where jtag->'tag1' between 1 and 30") + tdSql.checkRows(3) + tdSql.query("select * from jsons1 where jtag->'tag1' between 'femail' and 'beijing'") + tdSql.checkRows(2) # # # test with tbname/normal column tdSql.query("select * from jsons1 where tbname = 'jsons1_1'") @@ -362,6 +356,7 @@ class TDTestCase: # # # test where condition like # open + # syntax error #tdSql.query("select *,tbname from jsons1 where jtag->'tag2' like 'bei%'") #tdSql.checkRows(2) #tdSql.query("select *,tbname from jsons1 where jtag->'tag1' like 'fe%' and jtag->'tag2' is not null")