diff --git a/src/client/inc/tscLocalMerge.h b/src/client/inc/tscLocalMerge.h index 5baa66a9e0229f35c431cea7a0d2dbb9e2ffb0e2..0af8c8b5768c844954a072521e67a3cb6e914ac3 100644 --- a/src/client/inc/tscLocalMerge.h +++ b/src/client/inc/tscLocalMerge.h @@ -64,9 +64,8 @@ typedef struct SLocalReducer { SColumnModel * resColModel; tExtMemBuffer ** pExtMemBuffer; // disk-based buffer SFillInfo* pFillInfo; // interpolation support structure - char * pFinalRes; // result data after interpo - tFilePage * discardData; - SResultInfo * pResInfo; + char* pFinalRes; // result data after interpo + tFilePage* discardData; bool discard; int32_t offset; // limit offset value bool orderPrjOnSTable; // projection query on stable diff --git a/src/client/inc/tscSubquery.h b/src/client/inc/tscSubquery.h index bc01de110345e4c90cf5c15d3d7f6b010cb7308d..3406dcd8589e149f2a62040d1271fbfd40caaac7 100644 --- a/src/client/inc/tscSubquery.h +++ b/src/client/inc/tscSubquery.h @@ -23,7 +23,7 @@ extern "C" { #include "tscUtil.h" #include "tsclient.h" -void tscFetchDatablockFromSubquery(SSqlObj* pSql); +void tscFetchDatablockForSubquery(SSqlObj* pSql); void tscSetupOutputColumnIndex(SSqlObj* pSql); void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code); diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 2f918e1e9326a184205ae4e51515a1e8ad730a24..6628e3087456c52b2daf1b9ef84efa9577ad4f36 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -228,6 +228,7 @@ void tscClearSubqueryInfo(SSqlCmd* pCmd); void tscFreeVgroupTableInfo(SArray* pVgroupTables); SArray* tscVgroupTableInfoClone(SArray* pVgroupTables); void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index); +void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo); int tscGetSTableVgroupInfo(SSqlObj* pSql, int32_t clauseIndex); int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo); diff --git a/src/client/inc/tschemautil.h b/src/client/inc/tschemautil.h index e5bdcecfa8c006f75575d5b197ed29c9cb65fbc9..f6dc45398f35c38598f3f3132b2f6f5601a4ed68 100644 --- a/src/client/inc/tschemautil.h +++ b/src/client/inc/tschemautil.h @@ -107,9 +107,6 @@ SSchema tscGetTbnameColumnSchema(); */ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size); -//todo tags value as well as the table id structure needs refactor -char *tsGetTagsValue(STableMeta *pMeta); - #ifdef __cplusplus } #endif diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 0e5a206c2762a63fdde8d17958e6505025c79280..1c2821638bd24d67c51985a9e67da26e83e768b8 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -339,9 +339,9 @@ typedef struct STscObj { } STscObj; typedef struct SSubqueryState { - int32_t numOfRemain; // the number of remain unfinished subquery - int32_t numOfSub; // the number of total sub-queries - uint64_t numOfRetrievedRows; // total number of points in this query + int32_t numOfRemain; // the number of remain unfinished subquery + int32_t numOfSub; // the number of total sub-queries + uint64_t numOfRetrievedRows; // total number of points in this query } SSubqueryState; typedef struct SSqlObj { @@ -431,14 +431,6 @@ void tscResetSqlCmdObj(SSqlCmd *pCmd, bool removeFromCache); */ void tscFreeSqlResult(SSqlObj *pSql); -/** - * only free part of resources allocated during query. - * TODO remove it later - * Note: this function is multi-thread safe. - * @param pObj - */ -void tscPartiallyFreeSqlObj(SSqlObj *pSql); - /** * free sql object, release allocated resource * @param pObj @@ -523,7 +515,6 @@ extern SRpcCorEpSet tscMgmtEpSet; extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); -int32_t tscCompareTidTags(const void* p1, const void* p2); void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables); #ifdef __cplusplus diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index a85a0ea570f6a3bb7f57c916ff43ac290254fed4..e70cd11fbe03b498d2caad93b086cfc33adf5669 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -176,7 +176,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo } if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) { - tscFetchDatablockFromSubquery(pSql); + tscFetchDatablockForSubquery(pSql); } else { tscProcessSql(pSql); } @@ -226,7 +226,7 @@ void taos_fetch_rows_a(TAOS_RES *taosa, __async_cb_func_t fp, void *param) { // handle the sub queries of join query if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) { - tscFetchDatablockFromSubquery(pSql); + tscFetchDatablockForSubquery(pSql); } else if (pRes->completed) { if(pCmd->command == TSDB_SQL_FETCH || (pCmd->command >= TSDB_SQL_SERV_STATUS && pCmd->command <= TSDB_SQL_CURRENT_USER)) { if (hasMoreVnodesToTry(pSql)) { // sequentially retrieve data from remain vnodes. diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 12d3b7dfd38e09b30aed6b8e66e56e7eead61034..e0bd76233399b389c75aba4c28c04bdb8b1ac0f8 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -99,7 +99,7 @@ typedef struct SSumInfo { // the attribute of hasResult is not needed since the num attribute would server as this purpose typedef struct SAvgInfo { double sum; - int64_t num; // num servers as the hasResult attribute in other struct + int64_t num; } SAvgInfo; typedef struct SStddevInfo { @@ -167,7 +167,13 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_INTERP) { *type = (int16_t)dataType; *bytes = (int16_t)dataBytes; - *interBytes = *bytes + sizeof(SResultInfo); + + if (functionId == TSDB_FUNC_INTERP) { + *interBytes = sizeof(SInterpInfoDetail); + } else { + *interBytes = 0; + } + return TSDB_CODE_SUCCESS; } @@ -175,21 +181,21 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI if (functionId == TSDB_FUNC_TID_TAG) { // todo use struct *type = TSDB_DATA_TYPE_BINARY; *bytes = (int16_t)(dataBytes + sizeof(int16_t) + sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t) + VARSTR_HEADER_SIZE); - *interBytes = *bytes; + *interBytes = 0; return TSDB_CODE_SUCCESS; } if (functionId == TSDB_FUNC_COUNT) { *type = TSDB_DATA_TYPE_BIGINT; *bytes = sizeof(int64_t); - *interBytes = *bytes; + *interBytes = 0; return TSDB_CODE_SUCCESS; } if (functionId == TSDB_FUNC_ARITHM) { *type = TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(double); - *interBytes = *bytes; + *interBytes = 0; return TSDB_CODE_SUCCESS; } @@ -298,7 +304,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI } else if (functionId == TSDB_FUNC_FIRST || functionId == TSDB_FUNC_LAST) { *type = (int16_t)dataType; *bytes = (int16_t)dataBytes; - *interBytes = dataBytes + sizeof(SResultInfo); + *interBytes = dataBytes; } else if (functionId == TSDB_FUNC_SPREAD) { *type = (int16_t)TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(double); @@ -310,7 +316,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI } else if (functionId == TSDB_FUNC_LEASTSQR) { *type = TSDB_DATA_TYPE_BINARY; *bytes = TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE; // string - *interBytes = *bytes + sizeof(SResultInfo); + *interBytes = *bytes; } else if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_LAST_DST) { *type = TSDB_DATA_TYPE_BINARY; *bytes = (int16_t)(dataBytes + sizeof(SFirstLastInfo)); @@ -334,28 +340,20 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI return TSDB_CODE_SUCCESS; } -void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf) { - assert(pResInfo->interResultBuf == NULL); - - pResInfo->bufLen = size; - pResInfo->superTableQ = superTable; - pResInfo->interResultBuf = buf; -} - // set the query flag to denote that query is completed static void no_next_step(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->complete = true; } static bool function_setup(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->initialized) { return false; } memset(pCtx->aOutputBuf, 0, (size_t)pCtx->outputBytes); - initResultInfo(pResInfo); + initResultInfo(pResInfo, pCtx->interBufBytes); return true; } @@ -367,7 +365,7 @@ static bool function_setup(SQLFunctionCtx *pCtx) { * @param pCtx */ static void function_finalizer(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult != DATA_SET_FLAG) { if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) { setVardataNull(pCtx->aOutputBuf, pCtx->outputType); @@ -431,7 +429,7 @@ static void count_function_f(SQLFunctionCtx *pCtx, int32_t index) { *((int64_t *)pCtx->aOutputBuf) += 1; // do not need it actually - SResultInfo *pInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); pInfo->hasResult = DATA_SET_FLAG; } @@ -592,8 +590,8 @@ static void sum_function(SQLFunctionCtx *pCtx) { do_sum(pCtx); // keep the result data in output buffer, not in the intermediate buffer - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { // set the flag for super table query SSumInfo *pSum = (SSumInfo *)pCtx->aOutputBuf; pSum->hasResult = DATA_SET_FLAG; @@ -604,8 +602,8 @@ static void sum_function_f(SQLFunctionCtx *pCtx, int32_t index) { do_sum_f(pCtx, index); // keep the result data in output buffer, not in the intermediate buffer - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { SSumInfo *pSum = (SSumInfo *)pCtx->aOutputBuf; pSum->hasResult = DATA_SET_FLAG; } @@ -615,8 +613,7 @@ static int32_t sum_merge_impl(const SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; GET_TRUE_DATA_TYPE(); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ); + assert(pCtx->stableQuery); for (int32_t i = 0; i < pCtx->size; ++i) { char * input = GET_INPUT_CHAR_INDEX(pCtx, i); @@ -661,7 +658,7 @@ static void sum_func_second_merge(SQLFunctionCtx *pCtx) { int32_t notNullElems = sum_merge_impl(pCtx); SET_VAL(pCtx, notNullElems, 1); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (notNullElems > 0) { pResInfo->hasResult = DATA_SET_FLAG; @@ -755,9 +752,9 @@ static void avg_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; // NOTE: keep the intermediate result into the interResultBuf - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf; + SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); double * pVal = &pAvgInfo->sum; if (pCtx->preAggVals.isSet) { @@ -800,8 +797,8 @@ static void avg_function(SQLFunctionCtx *pCtx) { } // keep the data into the final output buffer for super table query since this execution may be the last one - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SAvgInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); } } @@ -814,9 +811,9 @@ static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); // NOTE: keep the intermediate result into the interResultBuf - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf; + SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { pAvgInfo->sum += GET_INT8_VAL(pData); @@ -839,16 +836,16 @@ static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) { pResInfo->hasResult = DATA_SET_FLAG; // keep the data into the final output buffer for super table query since this execution may be the last one - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SAvgInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); } } static void avg_func_merge(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + assert(pCtx->stableQuery); - SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf; + SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); char * input = GET_INPUT_CHAR(pCtx); for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { @@ -864,12 +861,12 @@ static void avg_func_merge(SQLFunctionCtx *pCtx) { // if the data set hasResult is not set, the result is null if (pAvgInfo->num > 0) { pResInfo->hasResult = DATA_SET_FLAG; - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SAvgInfo)); + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); } } static void avg_func_second_merge(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); double *sum = (double*) pCtx->aOutputBuf; char * input = GET_INPUT_CHAR(pCtx); @@ -883,7 +880,7 @@ static void avg_func_second_merge(SQLFunctionCtx *pCtx) { *sum += pInput->sum; // keep the number of data into the temp buffer - *(int64_t *)pResInfo->interResultBuf += pInput->num; + *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo) += pInput->num; } } @@ -891,21 +888,21 @@ static void avg_func_second_merge(SQLFunctionCtx *pCtx) { * the average value is calculated in finalize routine, since current routine does not know the exact number of points */ static void avg_finalizer(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - if (GET_INT64_VAL(pResInfo->interResultBuf) <= 0) { + if (GET_INT64_VAL(GET_ROWCELL_INTERBUF(pResInfo)) <= 0) { setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); return; // empty table } - *(double *)pCtx->aOutputBuf = (*(double *)pCtx->aOutputBuf) / *(int64_t *)pResInfo->interResultBuf; + *(double *)pCtx->aOutputBuf = (*(double *)pCtx->aOutputBuf) / *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo); } else { // this is the secondary merge, only in the secondary merge, the input type is TSDB_DATA_TYPE_BINARY assert(pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_DOUBLE); - SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf; + SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); if (pAvgInfo->num == 0) { // all data are NULL or empty table setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); @@ -1116,11 +1113,11 @@ static void min_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; // set the flag for super table query - if (pResInfo->superTableQ) { + if (pCtx->stableQuery) { *(pCtx->aOutputBuf + pCtx->inputBytes) = DATA_SET_FLAG; } } @@ -1133,11 +1130,11 @@ static void max_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; // set the flag for super table query - if (pResInfo->superTableQ) { + if (pCtx->stableQuery) { *(pCtx->aOutputBuf + pCtx->inputBytes) = DATA_SET_FLAG; } } @@ -1148,8 +1145,7 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp GET_TRUE_DATA_TYPE(); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ); + assert(pCtx->stableQuery); for (int32_t i = 0; i < pCtx->size; ++i) { char *input = GET_INPUT_CHAR_INDEX(pCtx, i); @@ -1210,7 +1206,7 @@ static void min_func_merge(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); - if (notNullElems > 0) { // for super table query, SResultInfo is not used + if (notNullElems > 0) { // for super table query, SResultRowCellInfo is not used char *flag = pCtx->aOutputBuf + pCtx->inputBytes; *flag = DATA_SET_FLAG; } @@ -1221,7 +1217,7 @@ static void min_func_second_merge(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (notNullElems > 0) { pResInfo->hasResult = DATA_SET_FLAG; } @@ -1242,7 +1238,7 @@ static void max_func_second_merge(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, numOfElem, 1); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (numOfElem > 0) { pResInfo->hasResult = DATA_SET_FLAG; } @@ -1297,8 +1293,8 @@ static void max_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); minMax_function_f(pCtx, index, 0); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { char *flag = pCtx->aOutputBuf + pCtx->inputBytes; *flag = DATA_SET_FLAG; } @@ -1313,8 +1309,8 @@ static void min_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); minMax_function_f(pCtx, index, 1); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { char *flag = pCtx->aOutputBuf + pCtx->inputBytes; *flag = DATA_SET_FLAG; } @@ -1330,7 +1326,7 @@ static void min_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void stddev_function(SQLFunctionCtx *pCtx) { // the second stage to calculate standard deviation - SStddevInfo *pStd = GET_RES_INFO(pCtx)->interResultBuf; + SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); if (pStd->stage == 0) { // the first stage is to calculate average value avg_function(pCtx); @@ -1381,8 +1377,8 @@ static void stddev_function(SQLFunctionCtx *pCtx) { static void stddev_function_f(SQLFunctionCtx *pCtx, int32_t index) { // the second stage to calculate standard deviation - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SStddevInfo *pStd = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo); /* the first stage is to calculate average value */ if (pStd->stage == 0) { @@ -1433,8 +1429,8 @@ static void stddev_next_step(SQLFunctionCtx *pCtx) { * the stddevInfo and the average info struct share the same buffer area * And the position of each element in their struct is exactly the same matched */ - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SStddevInfo *pStd = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo); if (pStd->stage == 0) { /* @@ -1449,7 +1445,7 @@ static void stddev_next_step(SQLFunctionCtx *pCtx) { pResInfo->initialized = true; // set it initialized to avoid re-initialization // save average value into tmpBuf, for second stage scan - SAvgInfo *pAvg = pResInfo->interResultBuf; + SAvgInfo *pAvg = GET_ROWCELL_INTERBUF(pResInfo); pStd->avg = GET_DOUBLE_VAL(pCtx->aOutputBuf); assert((isnan(pAvg->sum) && pAvg->num == 0) || (pStd->num == pAvg->num && pStd->avg == pAvg->sum)); @@ -1459,7 +1455,7 @@ static void stddev_next_step(SQLFunctionCtx *pCtx) { } static void stddev_finalizer(SQLFunctionCtx *pCtx) { - SStddevInfo *pStd = (SStddevInfo *)GET_RES_INFO(pCtx)->interResultBuf; + SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); if (pStd->num <= 0) { setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); @@ -1505,7 +1501,7 @@ static void first_function(SQLFunctionCtx *pCtx) { TSKEY k = pCtx->ptsList[i]; DO_UPDATE_TAG_COLUMNS(pCtx, k); - SResultInfo *pInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); pInfo->hasResult = DATA_SET_FLAG; pInfo->complete = true; @@ -1532,7 +1528,7 @@ static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) { TSKEY ts = pCtx->ptsList[index]; DO_UPDATE_TAG_COLUMNS(pCtx, ts); - SResultInfo *pInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); pInfo->hasResult = DATA_SET_FLAG; pInfo->complete = true; // get the first not-null data, completed } @@ -1576,7 +1572,7 @@ static void first_dist_function(SQLFunctionCtx *pCtx) { first_data_assign_impl(pCtx, data, i); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; notNullElems++; @@ -1604,8 +1600,7 @@ static void first_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void first_dist_func_merge(SQLFunctionCtx *pCtx) { char *pData = GET_INPUT_CHAR(pCtx); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pCtx->size == 1 && pResInfo->superTableQ); + assert(pCtx->size == 1 && pCtx->stableQuery); SFirstLastInfo *pInput = (SFirstLastInfo *)(pData + pCtx->inputBytes); if (pInput->hasResult != DATA_SET_FLAG) { @@ -1620,8 +1615,8 @@ static void first_dist_func_merge(SQLFunctionCtx *pCtx) { } static void first_dist_func_second_merge(SQLFunctionCtx *pCtx) { - assert(pCtx->resultInfo->superTableQ); - + assert(pCtx->stableQuery); + char * pData = GET_INPUT_CHAR(pCtx); SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes); if (pInput->hasResult != DATA_SET_FLAG) { @@ -1668,7 +1663,7 @@ static void last_function(SQLFunctionCtx *pCtx) { TSKEY ts = pCtx->ptsList[i]; DO_UPDATE_TAG_COLUMNS(pCtx, ts); - SResultInfo *pInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); pInfo->hasResult = DATA_SET_FLAG; pInfo->complete = true; // set query completed on this column @@ -1691,7 +1686,7 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { TSKEY ts = pCtx->ptsList[index]; DO_UPDATE_TAG_COLUMNS(pCtx, ts); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; pResInfo->complete = true; // set query completed } @@ -1740,7 +1735,7 @@ static void last_dist_function(SQLFunctionCtx *pCtx) { last_data_assign_impl(pCtx, data, i); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; notNullElems++; @@ -1776,8 +1771,7 @@ static void last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void last_dist_func_merge(SQLFunctionCtx *pCtx) { char *pData = GET_INPUT_CHAR(pCtx); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pCtx->size == 1 && pResInfo->superTableQ); + assert(pCtx->size == 1 && pCtx->stableQuery); // the input data is null SFirstLastInfo *pInput = (SFirstLastInfo *)(pData + pCtx->inputBytes); @@ -1833,11 +1827,11 @@ static void last_row_function(SQLFunctionCtx *pCtx) { // assign the last element in current data block assignVal(pCtx->aOutputBuf, pData + (pCtx->size - 1) * pCtx->inputBytes, pCtx->inputBytes, pCtx->inputType); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; // set the result to final result buffer in case of super table query - if (pResInfo->superTableQ) { + if (pCtx->stableQuery) { SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); pInfo1->ts = pCtx->ptsList[pCtx->size - 1]; pInfo1->hasResult = DATA_SET_FLAG; @@ -1852,7 +1846,7 @@ static void last_row_function(SQLFunctionCtx *pCtx) { static void last_row_finalizer(SQLFunctionCtx *pCtx) { // do nothing at the first stage - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult != DATA_SET_FLAG) { if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) { setVardataNull(pCtx->aOutputBuf, pCtx->outputType); @@ -2044,8 +2038,8 @@ static int32_t resDataAscComparFn(const void *pLeft, const void *pRight) { static int32_t resDataDescComparFn(const void *pLeft, const void *pRight) { return -resDataAscComparFn(pLeft, pRight); } static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - STopBotInfo *pRes = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo); tValuePair **tvp = pRes->res; @@ -2135,18 +2129,18 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { * top/bottom use the intermediate result buffer to keep the intermediate result */ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); // only the first_stage_merge is directly written data into final output buffer - if (pResInfo->superTableQ && pCtx->currentStage != SECONDARY_STAGE_MERGE) { + if (pCtx->stableQuery && pCtx->currentStage != SECONDARY_STAGE_MERGE) { return (STopBotInfo*) pCtx->aOutputBuf; } else { // during normal table query and super table at the secondary_stage, result is written to intermediate buffer - return pResInfo->interResultBuf; + return GET_ROWCELL_INTERBUF(pResInfo); } } bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const char *minval, const char *maxval) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo == NULL) { return true; } @@ -2252,7 +2246,7 @@ static void top_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } } @@ -2270,7 +2264,7 @@ static void top_function_f(SQLFunctionCtx *pCtx, int32_t index) { do_top_function_add(pRes, (int32_t)pCtx->param[0].i64Key, pData, pCtx->ptsList[index], pCtx->inputType, &pCtx->tagInfo, NULL, 0); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } @@ -2285,8 +2279,7 @@ static void top_func_merge(SQLFunctionCtx *pCtx) { // remmap the input buffer may cause the struct pointer invalid, so rebuild the STopBotInfo is necessary buildTopBotStruct(pInput, pCtx); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1); + assert(pCtx->stableQuery && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1); STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); @@ -2314,7 +2307,7 @@ static void top_func_second_merge(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, pInput->num, pOutput->num); if (pOutput->num > 0) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } } @@ -2343,7 +2336,7 @@ static void bottom_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } } @@ -2359,7 +2352,7 @@ static void bottom_function_f(SQLFunctionCtx *pCtx, int32_t index) { do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64Key, pData, pCtx->ptsList[index], pCtx->inputType, &pCtx->tagInfo, NULL, 0); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } @@ -2374,8 +2367,7 @@ static void bottom_func_merge(SQLFunctionCtx *pCtx) { // remmap the input buffer may cause the struct pointer invalid, so rebuild the STopBotInfo is necessary buildTopBotStruct(pInput, pCtx); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1); + assert(pCtx->stableQuery && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1); STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); @@ -2403,16 +2395,16 @@ static void bottom_func_second_merge(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, pInput->num, pOutput->num); if (pOutput->num > 0) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } } static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); // data in temporary list is less than the required number of results, not enough qualified number of results - STopBotInfo *pRes = pResInfo->interResultBuf; + STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo); if (pRes->num == 0) { // no result assert(pResInfo->hasResult != DATA_SET_FLAG); // TODO: @@ -2443,8 +2435,8 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx) { } // in the first round, get the min-max value of all involved data - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); SET_DOUBLE_VAL(&pInfo->minval, DBL_MAX); SET_DOUBLE_VAL(&pInfo->maxval, -DBL_MAX); pInfo->numOfElems = 0; @@ -2455,8 +2447,8 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx) { static void percentile_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - SResultInfo * pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); // the first stage, only acquire the min/max value if (pInfo->stage == 0) { @@ -2546,9 +2538,9 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { return; } - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo *pInfo = (SPercentileInfo *)pResInfo->interResultBuf; + SPercentileInfo *pInfo = (SPercentileInfo *)GET_ROWCELL_INTERBUF(pResInfo); if (pInfo->stage == 0) { // TODO extract functions @@ -2595,8 +2587,8 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void percentile_finalizer(SQLFunctionCtx *pCtx) { double v = pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i64Key : pCtx->param[0].dKey; - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - tMemBucket * pMemBucket = ((SPercentileInfo *)pResInfo->interResultBuf)->pMemBucket; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + tMemBucket * pMemBucket = ((SPercentileInfo *)GET_ROWCELL_INTERBUF(pResInfo))->pMemBucket; if (pMemBucket->total > 0) { // check for null *(double *)pCtx->aOutputBuf = getPercentile(pMemBucket, v); @@ -2609,8 +2601,8 @@ static void percentile_finalizer(SQLFunctionCtx *pCtx) { } static void percentile_next_step(SQLFunctionCtx *pCtx) { - SResultInfo * pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); if (pInfo->stage == 0) { // all data are null, set it completed @@ -2627,12 +2619,12 @@ static void percentile_next_step(SQLFunctionCtx *pCtx) { ////////////////////////////////////////////////////////////////////////////////// static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo->superTableQ && pCtx->currentStage != SECONDARY_STAGE_MERGE) { + if (pCtx->stableQuery && pCtx->currentStage != SECONDARY_STAGE_MERGE) { return (SAPercentileInfo*) pCtx->aOutputBuf; } else { - return pResInfo->interResultBuf; + return GET_ROWCELL_INTERBUF(pResInfo); } } @@ -2651,7 +2643,7 @@ static bool apercentile_function_setup(SQLFunctionCtx *pCtx) { static void apercentile_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - SResultInfo * pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo *pInfo = getAPerctInfo(pCtx); for (int32_t i = 0; i < pCtx->size; ++i) { @@ -2704,8 +2696,8 @@ static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { return; } - SResultInfo * pResInfo = GET_RES_INFO(pCtx); - SAPercentileInfo *pInfo = getAPerctInfo(pCtx); // pResInfo->interResultBuf; + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SAPercentileInfo *pInfo = getAPerctInfo(pCtx); double v = 0; switch (pCtx->inputType) { @@ -2736,8 +2728,8 @@ static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void apercentile_func_merge(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + assert(pCtx->stableQuery); SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_CHAR(pCtx); @@ -2794,7 +2786,7 @@ static void apercentile_func_second_merge(SQLFunctionCtx *pCtx) { pOutput->pHisto = pRes; } - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; SET_VAL(pCtx, 1, 1); } @@ -2802,8 +2794,8 @@ static void apercentile_func_second_merge(SQLFunctionCtx *pCtx) { static void apercentile_finalizer(SQLFunctionCtx *pCtx) { double v = (pCtx->param[0].nType == TSDB_DATA_TYPE_INT) ? pCtx->param[0].i64Key : pCtx->param[0].dKey; - SResultInfo * pResInfo = GET_RES_INFO(pCtx); - SAPercentileInfo *pOutput = pResInfo->interResultBuf; + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SAPercentileInfo *pOutput = GET_ROWCELL_INTERBUF(pResInfo); if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { if (pResInfo->hasResult == DATA_SET_FLAG) { // check for null @@ -2840,8 +2832,8 @@ static bool leastsquares_function_setup(SQLFunctionCtx *pCtx) { return false; } - SResultInfo * pResInfo = GET_RES_INFO(pCtx); - SLeastsquareInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SLeastsquareInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); // 2*3 matrix pInfo->startVal = pCtx->param[0].dKey; @@ -2867,8 +2859,8 @@ static bool leastsquares_function_setup(SQLFunctionCtx *pCtx) { } static void leastsquares_function(SQLFunctionCtx *pCtx) { - SResultInfo * pResInfo = GET_RES_INFO(pCtx); - SLeastsquareInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SLeastsquareInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); double(*param)[3] = pInfo->mat; double x = pInfo->startVal; @@ -2938,8 +2930,8 @@ static void leastsquares_function_f(SQLFunctionCtx *pCtx, int32_t index) { return; } - SResultInfo * pResInfo = GET_RES_INFO(pCtx); - SLeastsquareInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SLeastsquareInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); double(*param)[3] = pInfo->mat; @@ -2988,8 +2980,8 @@ static void leastsquares_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void leastsquares_finalizer(SQLFunctionCtx *pCtx) { // no data in query - SResultInfo * pResInfo = GET_RES_INFO(pCtx); - SLeastsquareInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SLeastsquareInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); if (pInfo->num == 0) { if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) { @@ -3054,7 +3046,7 @@ static void col_project_function(SQLFunctionCtx *pCtx) { } static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pCtx->numOfParams == 2) { // the number of output rows should not affect the final number of rows, so set it to be 0 return; } @@ -3486,7 +3478,7 @@ static bool spread_function_setup(SQLFunctionCtx *pCtx) { return false; } - SSpreadInfo *pInfo = GET_RES_INFO(pCtx)->interResultBuf; + SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); // this is the server-side setup function in client-side, the secondary merge do not need this procedure if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { @@ -3501,8 +3493,8 @@ static bool spread_function_setup(SQLFunctionCtx *pCtx) { } static void spread_function(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SSpreadInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); int32_t numOfElems = 0; @@ -3568,8 +3560,8 @@ static void spread_function(SQLFunctionCtx *pCtx) { } // keep the data into the final output buffer for super table query since this execution may be the last one - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SSpreadInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); } } @@ -3581,8 +3573,8 @@ static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SSpreadInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); double val = 0.0; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { @@ -3611,16 +3603,16 @@ static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) { pResInfo->hasResult = DATA_SET_FLAG; pInfo->hasResult = DATA_SET_FLAG; - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SSpreadInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); } } void spread_func_merge(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + assert(pCtx->stableQuery); - SSpreadInfo *pResData = pResInfo->interResultBuf; + SSpreadInfo *pResData = GET_ROWCELL_INTERBUF(pResInfo); int32_t notNullElems = 0; for (int32_t i = 0; i < pCtx->size; ++i) { @@ -3644,7 +3636,7 @@ void spread_func_merge(SQLFunctionCtx *pCtx) { } if (notNullElems > 0) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SSpreadInfo)); + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); pResInfo->hasResult = DATA_SET_FLAG; } } @@ -3675,7 +3667,7 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { * here we do not check the input data types, because in case of metric query, * the type of intermediate data is binary */ - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); @@ -3690,7 +3682,7 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { assert((pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_DOUBLE) || (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)); - SSpreadInfo *pInfo = GET_RES_INFO(pCtx)->interResultBuf; + SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); if (pInfo->hasResult != DATA_SET_FLAG) { setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); return; @@ -3714,8 +3706,8 @@ static bool twa_function_setup(SQLFunctionCtx *pCtx) { return false; } - SResultInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes; - STwaInfo * pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes; + STwaInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo); pInfo->lastKey = INT64_MIN; pInfo->type = pCtx->inputType; @@ -3754,8 +3746,8 @@ static void twa_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - STwaInfo * pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + STwaInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo); int32_t i = 0; @@ -3808,7 +3800,7 @@ static void twa_function(SQLFunctionCtx *pCtx) { pResInfo->hasResult = DATA_SET_FLAG; } - if (pResInfo->superTableQ) { + if (pCtx->stableQuery) { memcpy(pCtx->aOutputBuf, pInfo, sizeof(STwaInfo)); } @@ -3825,8 +3817,8 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { TSKEY *primaryKey = pCtx->ptsList; - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - STwaInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); if (pInfo->lastKey == INT64_MIN) { pInfo->lastKey = pCtx->nStartQueryTimestamp; @@ -3848,14 +3840,13 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { // pCtx->numOfIteratedElems += 1; pResInfo->hasResult = DATA_SET_FLAG; - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(STwaInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(STwaInfo)); } } static void twa_func_merge(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ); + assert(pCtx->stableQuery); STwaInfo *pBuf = (STwaInfo *)pCtx->aOutputBuf; char * indicator = pCtx->aInputElemBuf; @@ -3895,16 +3886,16 @@ static void twa_func_merge(SQLFunctionCtx *pCtx) { */ void twa_function_copy(SQLFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - memcpy(pResInfo->interResultBuf, pCtx->aInputElemBuf, (size_t)pCtx->inputBytes); + memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->aInputElemBuf, (size_t)pCtx->inputBytes); pResInfo->hasResult = ((STwaInfo *)pCtx->aInputElemBuf)->hasResult; } void twa_function_finalizer(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - STwaInfo *pInfo = (STwaInfo *)pResInfo->interResultBuf; + STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo); assert(pInfo->EKey >= pInfo->lastKey && pInfo->hasResult == pResInfo->hasResult); if (pInfo->hasResult != DATA_SET_FLAG) { @@ -3932,8 +3923,8 @@ void twa_function_finalizer(SQLFunctionCtx *pCtx) { */ static void interp_function(SQLFunctionCtx *pCtx) { // at this point, the value is existed, return directly - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SInterpInfoDetail* pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SInterpInfoDetail* pInfo = GET_ROWCELL_INTERBUF(pResInfo); if (pCtx->size == 1) { char *pData = GET_INPUT_CHAR(pCtx); @@ -4008,7 +3999,6 @@ static void interp_function(SQLFunctionCtx *pCtx) { } } } - } SET_VAL(pCtx, pCtx->size, 1); @@ -4019,8 +4009,8 @@ static bool ts_comp_function_setup(SQLFunctionCtx *pCtx) { return false; // not initialized since it has been initialized } - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - STSCompInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); pInfo->pTSBuf = tsBufCreate(false, pCtx->order); pInfo->pTSBuf->tsOrder = pCtx->order; @@ -4028,8 +4018,8 @@ static bool ts_comp_function_setup(SQLFunctionCtx *pCtx) { } static void ts_comp_function(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - STSBuf * pTSbuf = ((STSCompInfo *)(pResInfo->interResultBuf))->pTSBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + STSBuf * pTSbuf = ((STSCompInfo *)(GET_ROWCELL_INTERBUF(pResInfo)))->pTSBuf; const char *input = GET_INPUT_CHAR(pCtx); @@ -4053,8 +4043,8 @@ static void ts_comp_function_f(SQLFunctionCtx *pCtx, int32_t index) { return; } - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - STSCompInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); STSBuf *pTSbuf = pInfo->pTSBuf; @@ -4065,9 +4055,9 @@ static void ts_comp_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void ts_comp_finalize(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - STSCompInfo *pInfo = pResInfo->interResultBuf; + STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); STSBuf * pTSbuf = pInfo->pTSBuf; tsBufFlush(pTSbuf); @@ -4116,8 +4106,8 @@ static bool rate_function_setup(SQLFunctionCtx *pCtx) { return false; } - SResultInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes; - SRateInfo * pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes; + SRateInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo); pInfo->CorrectionValue = 0; pInfo->firstKey = INT64_MIN; @@ -4136,8 +4126,8 @@ static bool rate_function_setup(SQLFunctionCtx *pCtx) { static void rate_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); TSKEY *primaryKey = pCtx->ptsList; tscDebug("%p rate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); @@ -4200,8 +4190,8 @@ static void rate_function(SQLFunctionCtx *pCtx) { } // keep the data into the final output buffer for super table query since this execution may be the last one - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); } } @@ -4212,8 +4202,8 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { } // NOTE: keep the intermediate result into the interResultBuf - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); TSKEY *primaryKey = pCtx->ptsList; int64_t v = 0; @@ -4257,20 +4247,18 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { pResInfo->hasResult = DATA_SET_FLAG; // keep the data into the final output buffer for super table query since this execution may be the last one - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); } } static void rate_func_merge(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ); + assert(pCtx->stableQuery); tscDebug("rate_func_merge() size:%d", pCtx->size); - //SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; SRateInfo *pBuf = (SRateInfo *)pCtx->aOutputBuf; char *indicator = pCtx->aInputElemBuf; @@ -4303,8 +4291,8 @@ static void rate_func_merge(SQLFunctionCtx *pCtx) { static void rate_func_copy(SQLFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - memcpy(pResInfo->interResultBuf, pCtx->aInputElemBuf, (size_t)pCtx->inputBytes); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->aInputElemBuf, (size_t)pCtx->inputBytes); pResInfo->hasResult = ((SRateInfo*)pCtx->aInputElemBuf)->hasResult; SRateInfo* pRateInfo = (SRateInfo*)pCtx->aInputElemBuf; @@ -4315,8 +4303,8 @@ static void rate_func_copy(SQLFunctionCtx *pCtx) { static void rate_finalizer(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); tscDebug("%p isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " hasResult:%d", pCtx, pRateInfo->isIRate, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, pRateInfo->hasResult); @@ -4341,8 +4329,8 @@ static void rate_finalizer(SQLFunctionCtx *pCtx) { static void irate_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); TSKEY *primaryKey = pCtx->ptsList; tscDebug("%p irate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); @@ -4404,8 +4392,8 @@ static void irate_function(SQLFunctionCtx *pCtx) { } // keep the data into the final output buffer for super table query since this execution may be the last one - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); } } @@ -4416,8 +4404,8 @@ static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) { } // NOTE: keep the intermediate result into the interResultBuf - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); TSKEY *primaryKey = pCtx->ptsList; int64_t v = 0; @@ -4453,16 +4441,16 @@ static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) { pResInfo->hasResult = DATA_SET_FLAG; // keep the data into the final output buffer for super table query since this execution may be the last one - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); } } static void do_sumrate_merge(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + assert(pCtx->stableQuery); - SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); char * input = GET_INPUT_CHAR(pCtx); for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { @@ -4486,7 +4474,7 @@ static void do_sumrate_merge(SQLFunctionCtx *pCtx) { if (DATA_SET_FLAG == pRateInfo->hasResult) { pResInfo->hasResult = DATA_SET_FLAG; SET_VAL(pCtx, pRateInfo->num, 1); - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo)); + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); } } @@ -4501,10 +4489,10 @@ static void sumrate_func_second_merge(SQLFunctionCtx *pCtx) { } static void sumrate_finalizer(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - tscDebug("%p sumrate_finalizer() superTableQ:%d num:%" PRId64 " sum:%f hasResult:%d", pCtx, pResInfo->superTableQ, pRateInfo->num, pRateInfo->sum, pRateInfo->hasResult); + tscDebug("%p sumrate_finalizer() superTableQ:%d num:%" PRId64 " sum:%f hasResult:%d", pCtx, pCtx->stableQuery, pRateInfo->num, pRateInfo->sum, pRateInfo->hasResult); if (pRateInfo->hasResult != DATA_SET_FLAG) { setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index b4c3f3549b1576b3ff83300d4f10d9e07778530d..44dffab56f90e2d88604ee96a6cc1ddc8a5cfbef 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -49,82 +49,6 @@ typedef struct SCreateBuilder { } SCreateBuilder; static void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnName, int16_t type, size_t valueLength); -static int32_t getToStringLength(const char *pData, int32_t length, int32_t type) { - char buf[512] = {0}; - - int32_t len = 0; - int32_t MAX_BOOL_TYPE_LENGTH = 5; // max(strlen("true"), strlen("false")); - switch (type) { - case TSDB_DATA_TYPE_BINARY: - return length; - case TSDB_DATA_TYPE_NCHAR: - return length; - case TSDB_DATA_TYPE_DOUBLE: { - double dv = 0; - dv = GET_DOUBLE_VAL(pData); - len = sprintf(buf, "%lf", dv); - if (strncasecmp("nan", buf, 3) == 0) { - len = 4; - } - } break; - case TSDB_DATA_TYPE_FLOAT: { - float fv = 0; - fv = GET_FLOAT_VAL(pData); - len = sprintf(buf, "%f", fv); - if (strncasecmp("nan", buf, 3) == 0) { - len = 4; - } - } break; - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - len = sprintf(buf, "%" PRId64, *(int64_t *)pData); - break; - case TSDB_DATA_TYPE_BOOL: - len = MAX_BOOL_TYPE_LENGTH; - break; - default: - len = sprintf(buf, "%d", *(int32_t *)pData); - break; - }; - return len; -} - -/* - * we need to convert all data into string, so we need to sprintf all kinds of - * non-string data into string, and record its length to get the right - * maximum length. The length may be less or greater than its original binary length: - * For example: - * length((short) 1) == 1, less than sizeof(short) - * length((uint64_t) 123456789011) > 12, greater than sizsof(uint64_t) - */ -static int32_t tscMaxLengthOfTagsFields(SSqlObj *pSql) { - STableMeta *pMeta = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0)->pTableMeta; - - if (pMeta->tableType == TSDB_SUPER_TABLE || pMeta->tableType == TSDB_NORMAL_TABLE || - pMeta->tableType == TSDB_STREAM_TABLE) { - return 0; - } - - char * pTagValue = tsGetTagsValue(pMeta); - SSchema *pTagsSchema = tscGetTableTagSchema(pMeta); - - int32_t len = getToStringLength(pTagValue, pTagsSchema[0].bytes, pTagsSchema[0].type); - - pTagValue += pTagsSchema[0].bytes; - int32_t numOfTags = tscGetNumOfTags(pMeta); - - for (int32_t i = 1; i < numOfTags; ++i) { - int32_t tLen = getToStringLength(pTagValue, pTagsSchema[i].bytes, pTagsSchema[i].type); - if (len < tLen) { - len = tLen; - } - - pTagValue += pTagsSchema[i].bytes; - } - - return len; -} - static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { SSqlRes *pRes = &pSql->res; @@ -186,8 +110,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { return 0; } - // the following is handle display tags value for meters created according to metric - char *pTagValue = tsGetTagsValue(pMeta); + // the following is handle display tags for table created according to super table for (int32_t i = numOfRows; i < totalNumOfRows; ++i) { // field name TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0); @@ -219,8 +142,6 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { char *target = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i; const char *src = "TAG"; STR_WITH_MAXSIZE_TO_VARSTR(target, src, pField->bytes); - - pTagValue += pSchema[i].bytes; } return 0; @@ -286,10 +207,10 @@ static int32_t tscProcessDescribeTable(SSqlObj *pSql) { const int32_t TYPE_COLUMN_LENGTH = 16; const int32_t NOTE_COLUMN_MIN_LENGTH = 8; - int32_t noteFieldLen = tscMaxLengthOfTagsFields(pSql); - if (noteFieldLen == 0) { - noteFieldLen = NOTE_COLUMN_MIN_LENGTH; - } + int32_t noteFieldLen = NOTE_COLUMN_MIN_LENGTH;//tscMaxLengthOfTagsFields(pSql); +// if (noteFieldLen == 0) { +// noteFieldLen = NOTE_COLUMN_MIN_LENGTH; +// } int32_t rowLen = tscBuildTableSchemaResultFields(pSql, NUM_OF_DESC_TABLE_COLUMNS, TYPE_COLUMN_LENGTH, noteFieldLen); tscFieldInfoUpdateOffset(pQueryInfo); diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index ea073e8192e73f0f2bc1e3d8351decd9177ba403..c67edf5b5a8a026fc10d251218abcf96ab018359 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -99,12 +99,9 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc pCtx->param[1].i64Key = pQueryInfo->order.orderColId; } - SResultInfo *pResInfo = &pReducer->pResInfo[i]; - pResInfo->bufLen = pExpr->interBytes; - pResInfo->interResultBuf = calloc(1, (size_t) pResInfo->bufLen); - - pCtx->resultInfo = &pReducer->pResInfo[i]; - pCtx->resultInfo->superTableQ = true; + pCtx->interBufBytes = pExpr->interBytes; + pCtx->resultInfo = calloc(1, pCtx->interBufBytes + sizeof(SResultRowCellInfo)); + pCtx->stableQuery = true; } int16_t n = 0; @@ -345,7 +342,6 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd size_t numOfCols = tscSqlExprNumOfExprs(pQueryInfo); pReducer->pTempBuffer->num = 0; - pReducer->pResInfo = calloc(numOfCols, sizeof(SResultInfo)); tscCreateResPointerInfo(pRes, pQueryInfo); tscInitSqlContext(pCmd, pReducer, pDesc); @@ -489,13 +485,15 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { tscDebug("%p waiting for delete procedure, status: %d", pSql, status); } - pLocalReducer->pFillInfo = taosDestoryFillInfo(pLocalReducer->pFillInfo); + pLocalReducer->pFillInfo = taosDestroyFillInfo(pLocalReducer->pFillInfo); if (pLocalReducer->pCtx != NULL) { for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[i]; tVariantDestroy(&pCtx->tag); + taosTFree(pCtx->resultInfo); + if (pCtx->tagInfo.pTagCtxList != NULL) { taosTFree(pCtx->tagInfo.pTagCtxList); } @@ -509,15 +507,6 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { taosTFree(pLocalReducer->pTempBuffer); taosTFree(pLocalReducer->pResultBuf); - if (pLocalReducer->pResInfo != NULL) { - size_t num = tscSqlExprNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < num; ++i) { - taosTFree(pLocalReducer->pResInfo[i].interResultBuf); - } - - taosTFree(pLocalReducer->pResInfo); - } - if (pLocalReducer->pLoserTree) { taosTFree(pLocalReducer->pLoserTree->param); taosTFree(pLocalReducer->pLoserTree); @@ -1072,7 +1061,7 @@ static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx) continue; } - SResultInfo* pResInfo = GET_RES_INFO(&pCtx[j]); + SResultRowCellInfo* pResInfo = GET_RES_INFO(&pCtx[j]); if (maxOutput < pResInfo->numOfRes) { maxOutput = pResInfo->numOfRes; } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 01ff74e91d2807fe06c96c7f9179fc3d49c629d9..f7aaf830666e586d5ebd485011f3e94ddf47aeeb 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -65,7 +65,6 @@ static bool validateTagParams(tFieldList* pTagsList, tFieldList* pFieldList, SSq static int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* len); static void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nameLength); -static void getRevisedName(char* resultFieldName, int32_t functionId, int32_t maxLen, char* columnName); static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExprItem* pItem, bool finalResult); static int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pIdList, int16_t bytes, @@ -80,9 +79,9 @@ static void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo); static int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd); -static int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); +static int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); static int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); -static int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); +static int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); static int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExprItem* pItem); @@ -616,11 +615,13 @@ static bool isTopBottomQuery(SQueryInfo* pQueryInfo) { return false; } -int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { +int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { const char* msg1 = "invalid query expression"; const char* msg2 = "interval cannot be less than 10 ms"; const char* msg3 = "sliding cannot be used without interval"; + SSqlCmd* pCmd = &pSql->cmd; + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); @@ -660,7 +661,7 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ return TSDB_CODE_TSC_INVALID_SQL; } - if (parseSlidingClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { + if (parseSlidingClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -713,7 +714,7 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ return TSDB_CODE_TSC_INVALID_SQL; } - if (parseSlidingClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { + if (parseSlidingClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -771,13 +772,15 @@ int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQue return TSDB_CODE_SUCCESS; } -int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { +int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { const char* msg0 = "sliding value too small"; const char* msg1 = "sliding value no larger than the interval value"; const char* msg2 = "sliding value can not less than 1% of interval value"; - const char* msg3 = "does not support sliding when interval is natual month/year"; + const char* msg3 = "does not support sliding when interval is natural month/year"; + const char* msg4 = "sliding not support yet in ordinary query"; const static int32_t INTERVAL_SLIDING_FACTOR = 100; + SSqlCmd* pCmd = &pSql->cmd; STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); @@ -810,6 +813,10 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } + if (pQueryInfo->interval.sliding != pQueryInfo->interval.interval && pSql->pStream == NULL) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + } + return TSDB_CODE_SUCCESS; } @@ -1215,7 +1222,7 @@ static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* tscColumnListInsert(pQueryInfo->colList, &tsCol); } static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t exprIndex, tSQLExprItem* pItem) { - const char* msg1 = "invalid column name, or illegal column type"; + const char* msg1 = "invalid column name, illegal column type, or columns in arithmetic expression from two tables"; const char* msg2 = "invalid arithmetic expression in select clause"; const char* msg3 = "tag columns can not be used in arithmetic expression"; const char* msg4 = "columns from different table mixed up in arithmetic expression"; @@ -1629,16 +1636,15 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t } static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSchema* pSchema, SConvertFunc cvtFunc, - char* aliasName, int32_t resColIdx, SColumnIndex* pColIndex, bool finalResult) { + const char* name, int32_t resColIdx, SColumnIndex* pColIndex, bool finalResult) { const char* msg1 = "not support column types"; int16_t type = 0; int16_t bytes = 0; - char columnName[TSDB_COL_NAME_LEN] = {0}; int32_t functionID = cvtFunc.execFuncId; if (functionID == TSDB_FUNC_SPREAD) { - int32_t t1 = pSchema[pColIndex->columnIndex].type; + int32_t t1 = pSchema->type; if (t1 == TSDB_DATA_TYPE_BINARY || t1 == TSDB_DATA_TYPE_NCHAR || t1 == TSDB_DATA_TYPE_BOOL) { invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); return -1; @@ -1647,18 +1653,12 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS bytes = tDataTypeDesc[type].nSize; } } else { - type = pSchema[pColIndex->columnIndex].type; - bytes = pSchema[pColIndex->columnIndex].bytes; + type = pSchema->type; + bytes = pSchema->bytes; } - if (aliasName != NULL) { - tstrncpy(columnName, aliasName, sizeof(columnName)); - } else { - getRevisedName(columnName, cvtFunc.originFuncId, sizeof(columnName) - 1, pSchema[pColIndex->columnIndex].name); - } - SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, bytes, false); - tstrncpy(pExpr->aliasName, columnName, sizeof(pExpr->aliasName)); + tstrncpy(pExpr->aliasName, name, tListLen(pExpr->aliasName)); if (cvtFunc.originFuncId == TSDB_FUNC_LAST_ROW && cvtFunc.originFuncId != functionID) { pExpr->colInfo.flag |= TSDB_COL_NULL; @@ -1678,7 +1678,7 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS // if it is not in the final result, do not add it SColumnList ids = getColumnList(1, pColIndex->tableIndex, pColIndex->columnIndex); if (finalResult) { - insertResultField(pQueryInfo, resColIdx, &ids, bytes, (int8_t)type, columnName, pExpr); + insertResultField(pQueryInfo, resColIdx, &ids, bytes, (int8_t)type, pExpr->aliasName, pExpr); } else { tscColumnListInsert(pQueryInfo->colList, &(ids.ids[0])); } @@ -1686,6 +1686,23 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS return TSDB_CODE_SUCCESS; } +void setResultColName(char* name, tSQLExprItem* pItem, int32_t functionId, SStrToken* pToken) { + if (pItem->aliasName != NULL) { + tstrncpy(name, pItem->aliasName, TSDB_COL_NAME_LEN); + } else { + char uname[TSDB_COL_NAME_LEN] = {0}; + int32_t len = MIN(pToken->n + 1, TSDB_COL_NAME_LEN); + tstrncpy(uname, pToken->z, len); + + int32_t size = TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].aName) + 2 + 1; + char tmp[TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].aName) + 2 + 1] = {0}; + + snprintf(tmp, size, "%s(%s)", aAggs[functionId].aName, uname); + + tstrncpy(name, tmp, TSDB_COL_NAME_LEN); + } +} + int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExprItem* pItem, bool finalResult) { STableMetaInfo* pTableMetaInfo = NULL; int32_t optr = pItem->pNode->nSQLOptr; @@ -1943,8 +1960,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (pParamElem->pNode->nSQLOptr == TK_ALL) { - // select table.* + if (pParamElem->pNode->nSQLOptr == TK_ALL) { // select table.* SStrToken tmpToken = pParamElem->pNode->colInfo; if (getTableIndexByName(&tmpToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { @@ -1954,9 +1970,13 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); + char name[TSDB_COL_NAME_LEN] = {0}; for (int32_t j = 0; j < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++j) { index.columnIndex = j; - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex++, &index, finalResult) != 0) { + SStrToken t = {.z = pSchema[j].name, .n = (uint32_t)strnlen(pSchema[j].name, TSDB_COL_NAME_LEN)}; + setResultColName(name, pItem, cvtFunc.originFuncId, &t); + + if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[j], cvtFunc, name, colIndex++, &index, finalResult) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } } @@ -1967,14 +1987,18 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col } pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); // functions can not be applied to tags if ((index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) || (index.columnIndex < 0)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); } - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex + i, &index, finalResult) != 0) { + char name[TSDB_COL_NAME_LEN] = {0}; + + SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); + setResultColName(name, pItem, cvtFunc.originFuncId, &pParamElem->pNode->colInfo); + + if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, name, colIndex + i, &index, finalResult) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -2011,7 +2035,12 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col for (int32_t i = 0; i < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++i) { SColumnIndex index = {.tableIndex = j, .columnIndex = i}; - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex, &index, finalResult) != 0) { + + char name[TSDB_COL_NAME_LEN] = {0}; + SStrToken t = {.z = pSchema->name, .n = (uint32_t)strnlen(pSchema->name, TSDB_COL_NAME_LEN)}; + setResultColName(name, pItem, cvtFunc.originFuncId, &t); + + if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[index.columnIndex], cvtFunc, name, colIndex, &index, finalResult) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -2244,10 +2273,6 @@ void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nameLengt } } -void getRevisedName(char* resultFieldName, int32_t functionId, int32_t maxLen, char* columnName) { - snprintf(resultFieldName, maxLen, "%s(%s)", aAggs[functionId].aName, columnName); -} - static bool isTablenameToken(SStrToken* token) { SStrToken tmpToken = *token; SStrToken tableToken = {0}; @@ -2732,6 +2757,7 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) { int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd) { const char* msg1 = "too many columns in group by clause"; const char* msg2 = "invalid column name in group by clause"; + const char* msg3 = "columns from one table allowed as group by columns"; const char* msg7 = "not support group by expression"; const char* msg8 = "not allowed column type for group by"; const char* msg9 = "tags not allowed for table query"; @@ -2767,7 +2793,11 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - tableIndex = index.tableIndex; + if (tableIndex == COLUMN_INDEX_INITIAL_VAL) { + tableIndex = index.tableIndex; + } else if (tableIndex != index.tableIndex) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); pTableMeta = pTableMetaInfo->pTableMeta; @@ -3352,7 +3382,8 @@ int32_t doArithmeticExprToString(tSQLExpr* pExpr, char** exprString) { return TSDB_CODE_SUCCESS; } -static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type) { +static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, + int32_t* type, uint64_t* uid) { if (pExpr->nSQLOptr == TK_ID) { if (*type == NON_ARITHMEIC_EXPR) { *type = NORMAL_ARITHMETIC; @@ -3401,13 +3432,22 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQuer } // Not supported data type in arithmetic expression + uint64_t id = -1; for(int32_t i = 0; i < inc; ++i) { SSqlExpr* p1 = tscSqlExprGet(pQueryInfo, i + outputIndex); int16_t t = p1->resType; if (t == TSDB_DATA_TYPE_BINARY || t == TSDB_DATA_TYPE_NCHAR || t == TSDB_DATA_TYPE_BOOL || t == TSDB_DATA_TYPE_TIMESTAMP) { return TSDB_CODE_TSC_INVALID_SQL; } + + if (i == 0) { + id = p1->uid; + } else if (id != p1->uid){ + return TSDB_CODE_TSC_INVALID_SQL; + } } + + *uid = id; } return TSDB_CODE_SUCCESS; @@ -3419,13 +3459,16 @@ static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryI } tSQLExpr* pLeft = pExpr->pLeft; + uint64_t uidLeft = 0; + uint64_t uidRight = 0; + if (pLeft->nSQLOptr >= TK_PLUS && pLeft->nSQLOptr <= TK_REM) { int32_t ret = validateArithmeticSQLExpr(pCmd, pLeft, pQueryInfo, pList, type); if (ret != TSDB_CODE_SUCCESS) { return ret; } } else { - int32_t ret = validateSQLExpr(pCmd, pLeft, pQueryInfo, pList, type); + int32_t ret = validateSQLExpr(pCmd, pLeft, pQueryInfo, pList, type, &uidLeft); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -3438,10 +3481,15 @@ static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryI return ret; } } else { - int32_t ret = validateSQLExpr(pCmd, pRight, pQueryInfo, pList, type); + int32_t ret = validateSQLExpr(pCmd, pRight, pQueryInfo, pList, type, &uidRight); if (ret != TSDB_CODE_SUCCESS) { return ret; } + + // the expression not from the same table, return error + if (uidLeft != uidRight && uidLeft != 0 && uidRight != 0) { + return TSDB_CODE_TSC_INVALID_SQL; + } } return TSDB_CODE_SUCCESS; @@ -6103,7 +6151,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { } // set interval value - if (parseIntervalClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { + if (parseIntervalClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } else { if ((pQueryInfo->interval.interval > 0) && @@ -6311,7 +6359,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { } // set interval value - if (parseIntervalClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { + if (parseIntervalClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } else { if ((pQueryInfo->interval.interval > 0) && diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 0b1767dd2530d92e854c151f9be21985dce8d61c..441ad8d8382c667c164f7c797fc344cb9ab9118d 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -197,28 +197,6 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size return pTableMeta; } -/** - * the TableMeta data format in memory is as follows: - * - * +--------------------+ - * |STableMeta Body data| sizeof(STableMeta) - * +--------------------+ - * |Schema data | numOfTotalColumns * sizeof(SSchema) - * +--------------------+ - * |Tags data | tag_col_1.bytes + tag_col_2.bytes + .... - * +--------------------+ - * - * @param pTableMeta - * @return - */ -char* tsGetTagsValue(STableMeta* pTableMeta) { - int32_t offset = 0; -// int32_t numOfTotalCols = pTableMeta->numOfColumns + pTableMeta->numOfTags; -// uint32_t offset = sizeof(STableMeta) + numOfTotalCols * sizeof(SSchema); - - return ((char*)pTableMeta + offset); -} - // todo refactor UNUSED_FUNC static FORCE_INLINE char* skipSegments(char* input, char delim, int32_t num) { for (int32_t i = 0; i < num; ++i) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index e10097ed127b756b16da008605fa9dbcd0069c5f..1584afe706e21d1a16e4ab914297aadaccbbf1b3 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -878,37 +878,20 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // compressed ts block pQueryMsg->tsOffset = htonl((int32_t)(pMsg - pCmd->payload)); - int32_t tsLen = 0; - int32_t numOfBlocks = 0; if (pQueryInfo->tsBuf != NULL) { - int32_t vnodeId = htonl(pQueryMsg->head.vgId); - STSVnodeBlockInfo *pBlockInfo = tsBufGetVnodeBlockInfo(pQueryInfo->tsBuf, vnodeId); - assert(QUERY_IS_JOIN_QUERY(pQueryInfo->type) && pBlockInfo != NULL); // this query should not be sent - - // todo refactor - if (fseek(pQueryInfo->tsBuf->f, pBlockInfo->offset, SEEK_SET) != 0) { - int code = TAOS_SYSTEM_ERROR(ferror(pQueryInfo->tsBuf->f)); - tscError("%p: fseek failed: %s", pSql, tstrerror(code)); + // note: here used the index instead of actual vnode id. + int32_t vnodeIndex = pTableMetaInfo->vgroupIndex; + int32_t code = dumpFileBlockByVnodeId(pQueryInfo->tsBuf, vnodeIndex, pMsg, &pQueryMsg->tsLen, &pQueryMsg->tsNumOfBlocks); + if (code != TSDB_CODE_SUCCESS) { return code; } - size_t s = fread(pMsg, 1, pBlockInfo->compLen, pQueryInfo->tsBuf->f); - if (s != pBlockInfo->compLen) { - int code = TAOS_SYSTEM_ERROR(ferror(pQueryInfo->tsBuf->f)); - tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code)); - return code; - } + pMsg += pQueryMsg->tsLen; - pMsg += pBlockInfo->compLen; - tsLen = pBlockInfo->compLen; - numOfBlocks = pBlockInfo->numOfBlocks; - } - - pQueryMsg->tsLen = htonl(tsLen); - pQueryMsg->tsNumOfBlocks = htonl(numOfBlocks); - if (pQueryInfo->tsBuf != NULL) { pQueryMsg->tsOrder = htonl(pQueryInfo->tsBuf->tsOrder); + pQueryMsg->tsLen = htonl(pQueryMsg->tsLen); + pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks); } int32_t msgLen = (int32_t)(pMsg - pCmd->payload); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 0cd463c4f570cc46ffb909412286bf6f5fc68051..de0d899f8c363b5a3dd226f3f5ab69eeb8fc3aff 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -32,11 +32,26 @@ typedef struct SInsertSupporter { static void freeJoinSubqueryObj(SSqlObj* pSql); static bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql); -static bool tsCompare(int32_t order, int64_t left, int64_t right) { +static int32_t tsCompare(int32_t order, int64_t left, int64_t right) { + if (left == right) { + return 0; + } + if (order == TSDB_ORDER_ASC) { - return left < right; + return left < right? -1:1; } else { - return left > right; + return left > right? -1:1; + } +} + +static void skipRemainValue(STSBuf* pTSBuf, tVariant* tag1) { + while (tsBufNextPos(pTSBuf)) { + STSElem el1 = tsBufGetElem(pTSBuf); + + int32_t res = tVariantCompare(el1.tag, tag1); + if (res != 0) { // it is a record with new tag + return; + } } } @@ -51,10 +66,10 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ SLimitVal* pLimit = &pQueryInfo->limit; int32_t order = pQueryInfo->order.order; - + SQueryInfo* pSubQueryInfo1 = tscGetQueryInfoDetail(&pSql->pSubs[0]->cmd, 0); SQueryInfo* pSubQueryInfo2 = tscGetQueryInfoDetail(&pSql->pSubs[1]->cmd, 0); - + pSubQueryInfo1->tsBuf = output1; pSubQueryInfo2->tsBuf = output2; @@ -88,59 +103,74 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ int64_t numOfInput1 = 1; int64_t numOfInput2 = 1; - while (1) { - STSElem elem1 = tsBufGetElem(pSupporter1->pTSBuf); - STSElem elem2 = tsBufGetElem(pSupporter2->pTSBuf); + while(1) { + STSElem elem = tsBufGetElem(pSupporter1->pTSBuf); -#ifdef _DEBUG_VIEW - tscInfo("%" PRId64 ", tags:%"PRId64" \t %" PRId64 ", tags:%"PRId64, elem1.ts, elem1.tag.i64Key, elem2.ts, elem2.tag.i64Key); -#endif + // no data in pSupporter1 anymore, jump out of loop + if (!tsBufIsValidElem(&elem)) { + break; + } - int32_t res = tVariantCompare(elem1.tag, elem2.tag); - if (res == -1 || (res == 0 && tsCompare(order, elem1.ts, elem2.ts))) { - if (!tsBufNextPos(pSupporter1->pTSBuf)) { - break; - } + // find the data in supporter2 with the same tag value + STSElem e2 = tsBufFindElemStartPosByTag(pSupporter2->pTSBuf, elem.tag); - numOfInput1++; - } else if ((res > 0) || (res == 0 && tsCompare(order, elem2.ts, elem1.ts))) { - if (!tsBufNextPos(pSupporter2->pTSBuf)) { - break; - } + /** + * there are elements in pSupporter2 with the same tag, continue + */ + tVariant tag1 = {0}; + tVariantAssign(&tag1, elem.tag); - numOfInput2++; - } else { - /* - * in case of stable query, limit/offset is not applied here. the limit/offset is applied to the - * final results which is acquired after the secondry merge of in the client. - */ - if (pLimit->offset == 0 || pQueryInfo->interval.interval > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) { - if (win->skey > elem1.ts) { - win->skey = elem1.ts; - } - - if (win->ekey < elem1.ts) { - win->ekey = elem1.ts; + if (tsBufIsValidElem(&e2)) { + while (1) { + STSElem elem1 = tsBufGetElem(pSupporter1->pTSBuf); + STSElem elem2 = tsBufGetElem(pSupporter2->pTSBuf); + + // data with current are exhausted + if (!tsBufIsValidElem(&elem1) || tVariantCompare(elem1.tag, &tag1) != 0) { + break; } - - tsBufAppend(output1, elem1.vnode, elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts)); - tsBufAppend(output2, elem2.vnode, elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts)); - } else { - pLimit->offset -= 1; - } + if (!tsBufIsValidElem(&elem2) || tVariantCompare(elem2.tag, &tag1) != 0) { // ignore all records with the same tag + skipRemainValue(pSupporter1->pTSBuf, &tag1); + break; + } - if (!tsBufNextPos(pSupporter1->pTSBuf)) { - break; - } + /* + * in case of stable query, limit/offset is not applied here. the limit/offset is applied to the + * final results which is acquired after the secondary merge of in the client. + */ + int32_t re = tsCompare(order, elem1.ts, elem2.ts); + if (re < 0) { + tsBufNextPos(pSupporter1->pTSBuf); + numOfInput1++; + } else if (re > 0) { + tsBufNextPos(pSupporter2->pTSBuf); + numOfInput2++; + } else { + if (pLimit->offset == 0 || pQueryInfo->interval.interval > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) { + if (win->skey > elem1.ts) { + win->skey = elem1.ts; + } + + if (win->ekey < elem1.ts) { + win->ekey = elem1.ts; + } + + tsBufAppend(output1, elem1.vnode, elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts)); + tsBufAppend(output2, elem2.vnode, elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts)); + } else { + pLimit->offset -= 1;//offset apply to projection? + } - numOfInput1++; + tsBufNextPos(pSupporter1->pTSBuf); + numOfInput1++; - if (!tsBufNextPos(pSupporter2->pTSBuf)) { - break; + tsBufNextPos(pSupporter2->pTSBuf); + numOfInput2++; + } } - - numOfInput2++; + } else { // no data in pSupporter2, ignore current data in pSupporter2 + skipRemainValue(pSupporter1->pTSBuf, &tag1); } } @@ -162,8 +192,9 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ TSKEY et = taosGetTimestampUs(); tscDebug("%p input1:%" PRId64 ", input2:%" PRId64 ", final:%" PRId64 " in %d vnodes for secondary query after ts blocks " - "intersecting, skey:%" PRId64 ", ekey:%" PRId64 ", numOfVnode:%d, elasped time:%"PRId64" us", pSql, numOfInput1, numOfInput2, output1->numOfTotal, - output1->numOfVnodes, win->skey, win->ekey, tsBufGetNumOfVnodes(output1), et - st); + "intersecting, skey:%" PRId64 ", ekey:%" PRId64 ", numOfVnode:%d, elapsed time:%" PRId64 " us", + pSql, numOfInput1, numOfInput2, output1->numOfTotal, output1->numOfVnodes, win->skey, win->ekey, + tsBufGetNumOfVnodes(output1), et - st); return output1->numOfTotal; } @@ -248,6 +279,68 @@ static UNUSED_FUNC bool needSecondaryQuery(SQueryInfo* pQueryInfo) { return false; } +static void filterVgroupTables(SQueryInfo* pQueryInfo, SArray* pVgroupTables) { + int32_t num = 0; + int32_t* list = NULL; + tsBufGetVnodeIdList(pQueryInfo->tsBuf, &num, &list); + + // The virtual node, of which all tables are disqualified after the timestamp intersection, + // is removed to avoid next stage query. + // TODO: If tables from some vnodes are not qualified for next stage query, discard them. + for (int32_t k = 0; k < taosArrayGetSize(pVgroupTables);) { + SVgroupTableInfo* p = taosArrayGet(pVgroupTables, k); + + bool found = false; + for (int32_t f = 0; f < num; ++f) { + if (p->vgInfo.vgId == list[f]) { + found = true; + break; + } + } + + if (!found) { + tscRemoveVgroupTableGroup(pVgroupTables, k); + } else { + k++; + } + } + + assert(taosArrayGetSize(pVgroupTables) > 0); + TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY); + + taosTFree(list); +} + +static SArray* buildVgroupTableByResult(SQueryInfo* pQueryInfo, SArray* pVgroupTables) { + int32_t num = 0; + int32_t* list = NULL; + tsBufGetVnodeIdList(pQueryInfo->tsBuf, &num, &list); + + size_t numOfGroups = taosArrayGetSize(pVgroupTables); + + SArray* pNew = taosArrayInit(num, sizeof(SVgroupTableInfo)); + + SVgroupTableInfo info; + for (int32_t i = 0; i < num; ++i) { + int32_t vnodeId = list[i]; + + for (int32_t j = 0; j < numOfGroups; ++j) { + SVgroupTableInfo* p1 = taosArrayGet(pVgroupTables, j); + if (p1->vgInfo.vgId == vnodeId) { + tscVgroupTableCopy(&info, p1); + break; + } + } + + taosArrayPush(pNew, &info); + } + + taosTFree(list); + TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY); + + return pNew; +} + /* * launch secondary stage query to fetch the result that contains timestamp in set */ @@ -322,12 +415,11 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { pQueryInfo->fieldsInfo = pSupporter->fieldsInfo; pQueryInfo->groupbyExpr = pSupporter->groupInfo; - SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); - assert(pNew->subState.numOfSub == 0 && pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1); + assert(pNew->subState.numOfSub == 0 && pNew->cmd.numOfClause == 1 && pQueryInfo->numOfTables == 1); - tscFieldInfoUpdateOffset(pNewQueryInfo); + tscFieldInfoUpdateOffset(pQueryInfo); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); pTableMetaInfo->pVgroupTables = pSupporter->pVgroupTables; pSupporter->exprList = NULL; @@ -341,7 +433,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { * during the timestamp intersection. */ pSupporter->limit = pQueryInfo->limit; - pNewQueryInfo->limit = pSupporter->limit; + pQueryInfo->limit = pSupporter->limit; SColumnIndex index = {.tableIndex = 0, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; SSchema* s = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, 0); @@ -356,7 +448,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { tscAddSpecialColumnForSelect(pQueryInfo, 0, functionId, &index, s, TSDB_COL_NORMAL); tscPrintSelectClause(pNew, 0); - tscFieldInfoUpdateOffset(pNewQueryInfo); + tscFieldInfoUpdateOffset(pQueryInfo); pExpr = tscSqlExprGet(pQueryInfo, 0); } @@ -371,39 +463,21 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { pExpr->numOfParams = 1; } - int32_t num = 0; - int32_t *list = NULL; - tsBufGetVnodeIdList(pNewQueryInfo->tsBuf, &num, &list); - - if (pTableMetaInfo->pVgroupTables != NULL) { - for(int32_t k = 0; k < taosArrayGetSize(pTableMetaInfo->pVgroupTables);) { - SVgroupTableInfo* p = taosArrayGet(pTableMetaInfo->pVgroupTables, k); - - bool found = false; - for(int32_t f = 0; f < num; ++f) { - if (p->vgInfo.vgId == list[f]) { - found = true; - break; - } - } - - if (!found) { - tscRemoveVgroupTableGroup(pTableMetaInfo->pVgroupTables, k); - } else { - k++; - } + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + assert(pTableMetaInfo->pVgroupTables != NULL); + if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { + SArray* p = buildVgroupTableByResult(pQueryInfo, pTableMetaInfo->pVgroupTables); + tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables); + pTableMetaInfo->pVgroupTables = p; + } else { + filterVgroupTables(pQueryInfo, pTableMetaInfo->pVgroupTables); } - - assert(taosArrayGetSize(pTableMetaInfo->pVgroupTables) > 0); - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY); } - taosTFree(list); - - size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList); + size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); tscDebug("%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, name:%s", - pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, taosArrayGetSize(pNewQueryInfo->exprList), - numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, pTableMetaInfo->name); + pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pQueryInfo->type, taosArrayGetSize(pQueryInfo->exprList), + numOfCols, pQueryInfo->fieldsInfo.numOfOutput, pTableMetaInfo->name); } //prepare the subqueries object failed, abort @@ -517,17 +591,29 @@ void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArr vgTables = taosArrayInit(4, sizeof(STableIdInfo)); info.itemList = vgTables; + + if (taosArrayGetSize(result) > 0) { + SVgroupTableInfo* prevGroup = taosArrayGet(result, taosArrayGetSize(result) - 1); + tscDebug("%p vgId:%d, tables:%"PRId64, pSql, prevGroup->vgInfo.vgId, taosArrayGetSize(prevGroup->itemList)); + } + taosArrayPush(result, &info); } - tscDebug("%p tid:%d, uid:%"PRIu64",vgId:%d added for vnode query", pSql, tt->tid, tt->uid, tt->vgId) STableIdInfo item = {.uid = tt->uid, .tid = tt->tid, .key = INT64_MIN}; taosArrayPush(vgTables, &item); + + tscTrace("%p tid:%d, uid:%"PRIu64",vgId:%d added", pSql, tt->tid, tt->uid, tt->vgId); prev = tt; } pTableMetaInfo->pVgroupTables = result; pTableMetaInfo->vgroupIndex = 0; + + if (taosArrayGetSize(result) > 0) { + SVgroupTableInfo* g = taosArrayGet(result, taosArrayGetSize(result) - 1); + tscDebug("%p vgId:%d, tables:%"PRId64, pSql, g->vgInfo.vgId, taosArrayGetSize(g->itemList)); + } } static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* pParent) { @@ -602,11 +688,11 @@ static bool checkForDuplicateTagVal(SSchema* pColSchema, SJoinSupporter* p1, SSq } static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pParentSql, SArray** s1, SArray** s2) { - tscDebug("%p all subqueries retrieve complete, do tags match", pParentSql); - SJoinSupporter* p1 = pParentSql->pSubs[0]->param; SJoinSupporter* p2 = pParentSql->pSubs[1]->param; + tscDebug("%p all subquery retrieve complete, do tags match, %d, %d", pParentSql, p1->num, p2->num); + // sort according to the tag value qsort(p1->pIdTagList, p1->num, p1->tagSize, tagValCompar); qsort(p2->pIdTagList, p2->num, p2->tagSize, tagValCompar); @@ -655,6 +741,19 @@ static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pPar qsort((*s1)->pData, t1, size, tidTagsCompar); qsort((*s2)->pData, t2, size, tidTagsCompar); +#if 0 + for(int32_t k = 0; k < t1; ++k) { + STidTags* p = (*s1)->pData + size * k; + printf("%d, tag:%s\n", p->vgId, ((tstr*)(p->tag))->data); + } + + for(int32_t k = 0; k < t1; ++k) { + STidTags* p = (*s2)->pData + size * k; + printf("%d, tag:%s\n", p->vgId, ((tstr*)(p->tag))->data); + } +#endif + + tscDebug("%p tags match complete, result: %"PRId64", %"PRId64, pParentSql, t1, t2); return TSDB_CODE_SUCCESS; } @@ -958,6 +1057,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR } } + assert(pState->numOfRemain > 0); if (atomic_sub_fetch_32(&pState->numOfRemain, 1) > 0) { tscDebug("%p sub:%p completed, remain:%d, total:%d", pParentSql, tres, pState->numOfRemain, pState->numOfSub); return; @@ -971,6 +1071,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR } // update the records for each subquery in parent sql object. + bool stableQuery = tscIsTwoStageSTableQuery(pQueryInfo, 0); for (int32_t i = 0; i < pState->numOfSub; ++i) { if (pParentSql->pSubs[i] == NULL) { tscDebug("%p %p sub:%d not retrieve data", pParentSql, NULL, i); @@ -984,7 +1085,10 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR pRes1->numOfRows, pRes1->numOfTotal); assert(pRes1->row < pRes1->numOfRows); } else { - pRes1->numOfClauseTotal += pRes1->numOfRows; + if (!stableQuery) { + pRes1->numOfClauseTotal += pRes1->numOfRows; + } + tscDebug("%p sub:%p index:%d numOfRows:%"PRId64" total:%"PRId64, pParentSql, pParentSql->pSubs[i], i, pRes1->numOfRows, pRes1->numOfTotal); } @@ -994,7 +1098,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR tscBuildResFromSubqueries(pParentSql); } -void tscFetchDatablockFromSubquery(SSqlObj* pSql) { +void tscFetchDatablockForSubquery(SSqlObj* pSql) { assert(pSql->subState.numOfSub >= 1); int32_t numOfFetch = 0; @@ -1056,11 +1160,22 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) { if (numOfFetch <= 0) { bool tryNextVnode = false; - SSqlObj* pp = pSql->pSubs[0]; - SQueryInfo* pi = tscGetQueryInfoDetail(&pp->cmd, 0); + bool orderedPrjQuery = false; + for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { + SSqlObj* pSub = pSql->pSubs[i]; + if (pSub == NULL) { + continue; + } + + SQueryInfo* p = tscGetQueryInfoDetail(&pSub->cmd, 0); + orderedPrjQuery = tscNonOrderedProjectionQueryOnSTable(p, 0); + if (orderedPrjQuery) { + break; + } + } // get the number of subquery that need to retrieve the next vnode. - if (tscNonOrderedProjectionQueryOnSTable(pi, 0)) { + if (orderedPrjQuery) { for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { SSqlObj* pSub = pSql->pSubs[i]; if (pSub != NULL && pSub->res.row >= pSub->res.numOfRows && pSub->res.completed) { @@ -1164,7 +1279,6 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; SSqlRes* pRes = &pSql->res; - tscDebug("%p all subquery response, retrieve data for subclause:%d", pSql, pCmd->clauseIndex); // the column transfer support struct has been built if (pRes->pColumnIndex != NULL) { @@ -1260,21 +1374,23 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { return; } - // wait for the other subqueries response from vnode - if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) { - return; + + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + // In case of consequence query from other vnode, do not wait for other query response here. + if (!(pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0))) { + if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) { + return; + } } tscSetupOutputColumnIndex(pParentSql); - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); /** * if the query is a continue query (vgroupIndex > 0 for projection query) for next vnode, do the retrieval of * data instead of returning to its invoker */ if (pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { -// pParentSql->subState.numOfRemain = pParentSql->subState.numOfSub; // reset the record value - pSql->fp = joinRetrieveFinalResCallback; // continue retrieve data pSql->cmd.command = TSDB_SQL_FETCH; tscProcessSql(pSql); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index ad518ab3f372b479175c7d1193d332e28ee00f0a..879eeeadedf03afc2a6dbd240c5500c801efc483 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -338,34 +338,6 @@ void tscFreeSqlResult(SSqlObj* pSql) { memset(&pSql->res, 0, sizeof(SSqlRes)); } -void tscPartiallyFreeSqlObj(SSqlObj* pSql) { - if (pSql == NULL || pSql->signature != pSql) { - return; - } - - SSqlCmd* pCmd = &pSql->cmd; - int32_t cmd = pCmd->command; - if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_LOCALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT || - cmd == TSDB_SQL_TABLE_JOIN_RETRIEVE) { - tscRemoveFromSqlList(pSql); - } - - // pSql->sqlstr will be used by tscBuildQueryStreamDesc -// if (pObj->signature == pObj) { - //pthread_mutex_lock(&pObj->mutex); - taosTFree(pSql->sqlstr); - //pthread_mutex_unlock(&pObj->mutex); -// } - - tscFreeSqlResult(pSql); - - taosTFree(pSql->pSubs); - pSql->subState.numOfSub = 0; - pSql->self = 0; - - tscResetSqlCmdObj(pCmd, false); -} - static void tscFreeSubobj(SSqlObj* pSql) { if (pSql->subState.numOfSub == 0) { return; @@ -434,22 +406,32 @@ void tscFreeSqlObj(SSqlObj* pSql) { tscDebug("%p start to free sqlObj", pSql); pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; + tscFreeSubobj(pSql); - tscPartiallyFreeSqlObj(pSql); + SSqlCmd* pCmd = &pSql->cmd; + int32_t cmd = pCmd->command; + if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_LOCALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT || + cmd == TSDB_SQL_TABLE_JOIN_RETRIEVE) { + tscRemoveFromSqlList(pSql); + } pSql->signature = NULL; pSql->fp = NULL; - - SSqlCmd* pCmd = &pSql->cmd; + taosTFree(pSql->sqlstr); + + taosTFree(pSql->pSubs); + pSql->subState.numOfSub = 0; + pSql->self = 0; + + tscFreeSqlResult(pSql); + tscResetSqlCmdObj(pCmd, false); memset(pCmd->payload, 0, (size_t)pCmd->allocSize); taosTFree(pCmd->payload); pCmd->allocSize = 0; - taosTFree(pSql->sqlstr); tsem_destroy(&pSql->rspSem); - free(pSql); } @@ -1714,6 +1696,17 @@ void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index) { taosArrayRemove(pVgroupTable, index); } +void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo) { + memset(info, 0, sizeof(SVgroupTableInfo)); + + info->vgInfo = pInfo->vgInfo; + for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) { + info->vgInfo.epAddr[j].fqdn = strdup(pInfo->vgInfo.epAddr[j].fqdn); + } + + info->itemList = taosArrayClone(pInfo->itemList); +} + SArray* tscVgroupTableInfoClone(SArray* pVgroupTables) { if (pVgroupTables == NULL) { return NULL; @@ -1725,14 +1718,8 @@ SArray* tscVgroupTableInfoClone(SArray* pVgroupTables) { SVgroupTableInfo info; for (size_t i = 0; i < num; i++) { SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i); - memset(&info, 0, sizeof(SVgroupTableInfo)); - - info.vgInfo = pInfo->vgInfo; - for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) { - info.vgInfo.epAddr[j].fqdn = strdup(pInfo->vgInfo.epAddr[j].fqdn); - } + tscVgroupTableCopy(&info, pInfo); - info.itemList = taosArrayClone(pInfo->itemList); taosArrayPush(pa, &info); } diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 9eb9924932c7757c3c999eb5afadd8c719dc16bc..ca1644c0a24472428097467ded7601294c4dcd87 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -108,7 +108,7 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32 break; } case TSDB_DATA_TYPE_BINARY: { // todo refactor, extract a method - pVar->pz = calloc(len, sizeof(char)); + pVar->pz = calloc(len + 1, sizeof(char)); memcpy(pVar->pz, pz, len); pVar->nLen = (int32_t)len; break; diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 9cdf81d5a3faf6403333b0cf61546590d4415b89..2c9d404e1d99ea4705363caba363b2d6380dca07 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -239,6 +239,7 @@ static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) { SHeartBeatMsg *pHBMsg = pMsg->rpcMsg.pCont; if (taosCheckVersion(pHBMsg->clientVer, version, 3) != TSDB_CODE_SUCCESS) { + rpcFreeCont(pRsp); return TSDB_CODE_TSC_INVALID_VERSION; // todo change the error code } diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index b474bea98717034c68ca4b3beb5a3a61288a064b..d7db897cdf9973019536a7b9bc8f8a4d7a8bc85d 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -40,6 +40,19 @@ typedef struct SGroupResInfo { int32_t rowId; } SGroupResInfo; +typedef struct SResultRowPool { + int32_t elemSize; + int32_t blockSize; + int32_t numOfElemPerBlock; + + struct { + int32_t blockIndex; + int32_t pos; + } position; + + SArray* pData; // SArray +} SResultRowPool; + typedef struct SSqlGroupbyExpr { int16_t tableIndex; SArray* columnInfo; // SArray, group by columns information @@ -48,14 +61,14 @@ typedef struct SSqlGroupbyExpr { int16_t orderType; // order by type: asc/desc } SSqlGroupbyExpr; -typedef struct SWindowResult { +typedef struct SResultRow { int32_t pageId; // pageId & rowId is the position of current result in disk-based output buffer int32_t rowId:15; bool closed:1; // this result status: closed or opened uint16_t numOfRows; // number of rows of current time window - SResultInfo* resultInfo; // For each result column, there is a resultInfo + SResultRowCellInfo* pCellInfo; // For each result column, there is a resultInfo union {STimeWindow win; char* key;}; // start key of current time window -} SWindowResult; +} SResultRow; /** * If the number of generated results is greater than this value, @@ -69,16 +82,14 @@ typedef struct SResultRec { } SResultRec; typedef struct SWindowResInfo { - SWindowResult* pResult; // result list - SHashObj* hashList; // hash list for quick access - int16_t type; // data type for hash key + SResultRow** pResult; // result list + int16_t type:8; // data type for hash key + int32_t size:24; // number of result set + int32_t threshold; // threshold to halt query and return the generated results. int32_t capacity; // max capacity int32_t curIndex; // current start active index - int32_t size; // number of result set int64_t startTime; // start time of the first time window for sliding query int64_t prevSKey; // previous (not completed) sliding window start key - int64_t threshold; // threshold to halt query and return the generated results. - int64_t interval; // time window interval } SWindowResInfo; typedef struct SColumnFilterElem { @@ -94,7 +105,7 @@ typedef struct SSingleColumnFilterInfo { SColumnFilterElem* pFilters; } SSingleColumnFilterInfo; -typedef struct STableQueryInfo { // todo merge with the STableQueryInfo struct +typedef struct STableQueryInfo { TSKEY lastKey; int32_t groupIndex; // group id in table list int16_t queryRangeSet; // denote if the query range is set, only available for interval query @@ -122,7 +133,9 @@ typedef struct SQueryCostInfo { uint32_t discardBlocks; uint64_t elapsedTime; uint64_t firstStageMergeTime; - uint64_t internalSupSize; + uint64_t winInfoSize; + uint64_t tableInfoSize; + uint64_t hashSize; uint64_t numOfTimeWindows; } SQueryCostInfo; @@ -155,11 +168,11 @@ typedef struct SQuery { typedef struct SQueryRuntimeEnv { jmp_buf env; - SResultInfo* resultInfo; // todo refactor to merge with SWindowResInfo + SResultRow* pResultRow; // todo refactor to merge with SWindowResInfo SQuery* pQuery; SQLFunctionCtx* pCtx; int32_t numOfRowsPerPage; - int16_t offset[TSDB_MAX_COLUMNS]; + uint16_t offset[TSDB_MAX_COLUMNS]; uint16_t scanFlag; // denotes reversed scan of data or not SFillInfo* pFillInfo; SWindowResInfo windowResInfo; @@ -175,6 +188,11 @@ typedef struct SQueryRuntimeEnv { int32_t interBufSize; // intermediate buffer sizse int32_t prevGroupId; // previous executed group id SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file + SHashObj* pResultRowHashTable; // quick locate the window object for each result + char* keyBuf; // window key buffer + SResultRowPool* pool; // window result object pool + + int32_t* rowCellInfoOffset;// offset value for each row result cell info } SQueryRuntimeEnv; enum { diff --git a/src/query/inc/qFill.h b/src/query/inc/qFill.h index 6d44fee09557fa8b8d73527677a2c7b238ecb42c..329ea9a789cf7e5c8963788c68981b63ccb234e3 100644 --- a/src/query/inc/qFill.h +++ b/src/query/inc/qFill.h @@ -71,7 +71,7 @@ SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_ void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp); -void* taosDestoryFillInfo(SFillInfo *pFillInfo); +void* taosDestroyFillInfo(SFillInfo *pFillInfo); void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h index bc8f9a5e23df72f82bcb3bb5140bec42fe426268..25da04710d08336796e6b63415e89d438c97fb42 100644 --- a/src/query/inc/qSqlparser.h +++ b/src/query/inc/qSqlparser.h @@ -223,13 +223,6 @@ typedef struct tSQLExprList { tSQLExprItem *a; /* One entry for each expression */ } tSQLExprList; -typedef struct tSQLExprListList { - int32_t nList; /* Number of expressions on the list */ - int32_t nAlloc; /* Number of entries allocated below */ - tSQLExprList **a; /* one entry for each row */ -} tSQLExprListList; - - /** * * @param yyp The parser diff --git a/src/query/inc/qTsbuf.h b/src/query/inc/qTsbuf.h index 6c2a955f47577a0da19f6520b2addc44e66b73ba..73b18f8915122305cca4a45604986f2e5593e274 100644 --- a/src/query/inc/qTsbuf.h +++ b/src/query/inc/qTsbuf.h @@ -135,6 +135,12 @@ int32_t tsBufGetNumOfVnodes(STSBuf* pTSBuf); void tsBufGetVnodeIdList(STSBuf* pTSBuf, int32_t* num, int32_t** vnodeId); +int32_t dumpFileBlockByVnodeId(STSBuf* pTSBuf, int32_t vnodeId, void* buf, int32_t* len, int32_t* numOfBlocks); + +STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, tVariant* pTag); + +bool tsBufIsValidElem(STSElem* pElem); + #ifdef __cplusplus } #endif diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index 32f26f66f5a46f98477db7e67c7a3a6d988fdfc8..5d649261a6e9fbf8b9eab44aab699ea792f86138 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -15,13 +15,22 @@ #ifndef TDENGINE_QUERYUTIL_H #define TDENGINE_QUERYUTIL_H +#define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid) \ + do { \ + assert(sizeof(_uid) == sizeof(uint64_t)); \ + *(uint64_t *)(_k) = (_uid); \ + memcpy((_k) + sizeof(uint64_t), (_ori), (_len)); \ + } while (0) + +#define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t)) + int32_t getOutputInterResultBufSize(SQuery* pQuery); -void clearTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* pOneOutputRes); -void copyTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* dst, const SWindowResult* src); +void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow); +void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src); +SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index); -int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, SQueryRuntimeEnv* pRuntimeEnv, int32_t size, - int32_t threshold, int16_t type); +int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, int32_t size, int32_t threshold, int16_t type); void cleanupTimeWindowInfo(SWindowResInfo* pWindowResInfo); void resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo* pWindowResInfo); @@ -33,9 +42,9 @@ void closeTimeWindow(SWindowResInfo* pWindowResInfo, int32_t slot); void closeAllTimeWindow(SWindowResInfo* pWindowResInfo); void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_t order); -static FORCE_INLINE SWindowResult *getWindowResult(SWindowResInfo *pWindowResInfo, int32_t slot) { +static FORCE_INLINE SResultRow *getResultRow(SWindowResInfo *pWindowResInfo, int32_t slot) { assert(pWindowResInfo != NULL && slot >= 0 && slot < pWindowResInfo->size); - return &pWindowResInfo->pResult[slot]; + return pWindowResInfo->pResult[slot]; } #define curTimeWindowIndex(_winres) ((_winres)->curIndex) @@ -43,9 +52,9 @@ static FORCE_INLINE SWindowResult *getWindowResult(SWindowResInfo *pWindowResInf bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot); -int32_t createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, size_t interBufSize); +int32_t initResultRow(SResultRow *pResultRow); -static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult, +static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SResultRow *pResult, tFilePage* page) { assert(pResult != NULL && pRuntimeEnv != NULL); @@ -62,4 +71,14 @@ bool notNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval); __filter_func_t *getRangeFilterFuncArray(int32_t type); __filter_func_t *getValueFilterFuncArray(int32_t type); +size_t getWindowResultSize(SQueryRuntimeEnv* pRuntimeEnv); + +SResultRowPool* initResultRowPool(size_t size); +SResultRow* getNewResultRow(SResultRowPool* p); +int64_t getResultRowPoolMemSize(SResultRowPool* p); +void* destroyResultRowPool(SResultRowPool* p); +int32_t getNumOfAllocatedResultRows(SResultRowPool* p); +int32_t getNumOfUsedResultRows(SResultRowPool* p); + + #endif // TDENGINE_QUERYUTIL_H diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 28b9a60102d41a11fa02fee1ca51cbf8e263bf3b..3bd4aad2766954709231f55ecdccfe03209a91bc 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -145,15 +145,14 @@ typedef struct SInterpInfoDetail { int8_t primaryCol; } SInterpInfoDetail; -typedef struct SResultInfo { +typedef struct SResultRowCellInfo { int8_t hasResult; // result generated, not NULL value - bool initialized; // output buffer has been initialized - bool complete; // query has completed - bool superTableQ; // is super table query - uint32_t bufLen; // buffer size - uint64_t numOfRes; // num of output result in current buffer - void* interResultBuf; // output result buffer -} SResultInfo; + bool initialized; // output buffer has been initialized + bool complete; // query has completed + uint16_t numOfRes; // num of output result in current buffer +} SResultRowCellInfo; + +#define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowCellInfo))) struct SQLFunctionCtx; @@ -175,9 +174,11 @@ typedef struct SQLFunctionCtx { int16_t inputBytes; int16_t outputType; - int16_t outputBytes; // size of results, determined by function and input column data type - bool hasNull; // null value exist in current block + int16_t outputBytes; // size of results, determined by function and input column data type + int32_t interBufBytes; // internal buffer size + bool hasNull; // null value exist in current block bool requireNull; // require null in some function + bool stableQuery; int16_t functionId; // function id void * aInputElemBuf; char * aOutputBuf; // final result output buffer, point to sdata->data @@ -189,7 +190,8 @@ typedef struct SQLFunctionCtx { void * ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ SQLPreAggVal preAggVals; tVariant tag; - SResultInfo *resultInfo; + + SResultRowCellInfo *resultInfo; SExtTagsInfo tagInfo; } SQLFunctionCtx; @@ -274,16 +276,16 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const cha (_r)->initialized = false; \ } while (0) -void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf); +//void setResultInfoBuf(SResultRowCellInfo *pResInfo, char* buf); -static FORCE_INLINE void initResultInfo(SResultInfo *pResInfo) { +static FORCE_INLINE void initResultInfo(SResultRowCellInfo *pResInfo, uint32_t bufLen) { pResInfo->initialized = true; // the this struct has been initialized flag pResInfo->complete = false; pResInfo->hasResult = false; pResInfo->numOfRes = 0; - memset(pResInfo->interResultBuf, 0, (size_t)pResInfo->bufLen); + memset(GET_ROWCELL_INTERBUF(pResInfo), 0, (size_t)bufLen); } #ifdef __cplusplus diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 7a7c1ee2d0fc9fe48824ee5fc4fd74e2c2f6834a..d65d01d1558795767a6ad497dea7312444f3686e 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -178,9 +178,9 @@ static void getNextTimeWindow(SQuery* pQuery, STimeWindow* tw) { // todo move to utility static int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *group); -static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult); -static void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult); -static void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pResultInfo); +static void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult); +static void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult); +static void resetMergeResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx *pCtx, SResultRow *pRow); static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId); static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo, @@ -255,7 +255,7 @@ int64_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv) { continue; } - SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); if (pResInfo != NULL && maxOutput < pResInfo->numOfRes) { maxOutput = pResInfo->numOfRes; } @@ -272,7 +272,7 @@ void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) { SQuery *pQuery = pRuntimeEnv->pQuery; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); int16_t functionId = pRuntimeEnv->pCtx[j].functionId; if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ || @@ -447,11 +447,11 @@ static bool hasNullValue(SColIndex* pColIndex, SDataStatis *pStatis, SDataStatis return true; } -static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, char *pData, - int16_t bytes, bool masterscan) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - int32_t *p1 = (int32_t *) taosHashGet(pWindowResInfo->hashList, pData, bytes); +static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, char *pData, + int16_t bytes, bool masterscan, uint64_t uid) { + SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid); + int32_t *p1 = + (int32_t *)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); if (p1 != NULL) { pWindowResInfo->curIndex = *p1; } else { @@ -461,41 +461,37 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin // more than the capacity, reallocate the resources if (pWindowResInfo->size >= pWindowResInfo->capacity) { - int64_t newCap = 0; + int64_t newCapacity = 0; if (pWindowResInfo->capacity > 10000) { - newCap = (int64_t)(pWindowResInfo->capacity * 1.25); + newCapacity = (int64_t)(pWindowResInfo->capacity * 1.25); } else { - newCap = (int64_t)(pWindowResInfo->capacity * 1.5); + newCapacity = (int64_t)(pWindowResInfo->capacity * 1.5); } - char *t = realloc(pWindowResInfo->pResult, (size_t)(newCap * sizeof(SWindowResult))); - pRuntimeEnv->summary.internalSupSize += (newCap - pWindowResInfo->capacity) * sizeof(SWindowResult); - pRuntimeEnv->summary.numOfTimeWindows += (newCap - pWindowResInfo->capacity); - + char *t = realloc(pWindowResInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); if (t == NULL) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - pWindowResInfo->pResult = (SWindowResult *)t; - - int32_t inc = (int32_t)newCap - pWindowResInfo->capacity; - memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, sizeof(SWindowResult) * inc); + pWindowResInfo->pResult = (SResultRow **)t; - pRuntimeEnv->summary.internalSupSize += (pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * inc; + int32_t inc = (int32_t)newCapacity - pWindowResInfo->capacity; + memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, POINTER_BYTES * inc); - for (int32_t i = pWindowResInfo->capacity; i < newCap; ++i) { - int32_t ret = createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - } + pWindowResInfo->capacity = (int32_t)newCapacity; + } - pWindowResInfo->capacity = (int32_t)newCap; + SResultRow *pResult = getNewResultRow(pRuntimeEnv->pool); + pWindowResInfo->pResult[pWindowResInfo->size] = pResult; + int32_t ret = initResultRow(pResult); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } // add a new result set for a new group pWindowResInfo->curIndex = pWindowResInfo->size++; - taosHashPut(pWindowResInfo->hashList, pData, bytes, (char *)&pWindowResInfo->curIndex, sizeof(int32_t)); + taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), + (char *)&pWindowResInfo->curIndex, sizeof(int32_t)); } // too many time window in query @@ -503,7 +499,7 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); } - return getWindowResult(pWindowResInfo, pWindowResInfo->curIndex); + return getResultRow(pWindowResInfo, pWindowResInfo->curIndex); } // get the correct time window according to the handled timestamp @@ -519,7 +515,7 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t } } else { int32_t slot = curTimeWindowIndex(pWindowResInfo); - SWindowResult* pWindowRes = getWindowResult(pWindowResInfo, slot); + SResultRow* pWindowRes = getResultRow(pWindowResInfo, slot); w = pWindowRes->win; } @@ -555,7 +551,7 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t return w; } -static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t sid, +static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t tid, int32_t numOfRowsPerPage) { if (pWindowRes->pageId != -1) { return 0; @@ -565,10 +561,10 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult // in the first scan, new space needed for results int32_t pageId = -1; - SIDList list = getDataBufPagesIdList(pResultBuf, sid); + SIDList list = getDataBufPagesIdList(pResultBuf, tid); if (taosArrayGetSize(list) == 0) { - pData = getNewDataBuf(pResultBuf, sid, &pageId); + pData = getNewDataBuf(pResultBuf, tid, &pageId); } else { SPageInfo* pi = getLastPageInfo(list); pData = getResBufPage(pResultBuf, pi->pageId); @@ -578,7 +574,7 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult // release current page first, and prepare the next one releaseResBufPageInfo(pResultBuf, pi); - pData = getNewDataBuf(pResultBuf, sid, &pageId); + pData = getNewDataBuf(pResultBuf, tid, &pageId); if (pData != NULL) { assert(pData->num == 0); // number of elements must be 0 for new allocated buffer } @@ -600,14 +596,13 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult return 0; } -static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, int32_t sid, +static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, SDataBlockInfo* pBockInfo, STimeWindow *win, bool masterscan, bool* newWind) { assert(win->skey <= win->ekey); SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&win->skey, - TSDB_KEYSIZE, masterscan); - if (pWindowRes == NULL) { + SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, pBockInfo->uid); + if (pResultRow == NULL) { *newWind = false; return masterscan? -1:0; @@ -616,23 +611,22 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes *newWind = true; // not assign result buffer yet, add new result buffer - if (pWindowRes->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, sid, pRuntimeEnv->numOfRowsPerPage); + if (pResultRow->pageId == -1) { + int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, pBockInfo->tid, pRuntimeEnv->numOfRowsPerPage); if (ret != TSDB_CODE_SUCCESS) { return -1; } } // set time window for current result - pWindowRes->win = (*win); - - setWindowResOutputBufInitCtx(pRuntimeEnv, pWindowRes); + pResultRow->win = (*win); + setWindowResOutputBufInitCtx(pRuntimeEnv, pResultRow); return TSDB_CODE_SUCCESS; } static bool getTimeWindowResStatus(SWindowResInfo *pWindowResInfo, int32_t slot) { assert(slot >= 0 && slot < pWindowResInfo->size); - return pWindowResInfo->pResult[slot].closed; + return pWindowResInfo->pResult[slot]->closed; } static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int16_t pos, @@ -691,7 +685,7 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe int64_t skey = TSKEY_INITIAL_VAL; for (i = 0; i < pWindowResInfo->size; ++i) { - SWindowResult *pResult = &pWindowResInfo->pResult[i]; + SResultRow *pResult = pWindowResInfo->pResult[i]; if (pResult->closed) { numOfClosed += 1; continue; @@ -715,7 +709,7 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe pWindowResInfo->curIndex = i; } - pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex].win.skey; + pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex]->win.skey; // the number of completed slots are larger than the threshold, return current generated results to client. if (numOfClosed > pWindowResInfo->threshold) { @@ -1024,8 +1018,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * bool hasTimeWindow = false; STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &win, masterScan, &hasTimeWindow) != - TSDB_CODE_SUCCESS) { + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) { taosTFree(sasArray); return; } @@ -1053,8 +1046,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * // null data, failed to allocate more memory buffer hasTimeWindow = false; - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &nextWin, masterScan, - &hasTimeWindow) != TSDB_CODE_SUCCESS) { + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &nextWin, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) { break; } @@ -1095,7 +1087,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * taosTFree(sasArray); } -static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pData, int16_t type, int16_t bytes) { +static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) { if (isNull(pData, type)) { // ignore the null value return -1; } @@ -1112,13 +1104,14 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat len = varDataLen(pData); } else if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); - qError("QInfo:%p group by not supported on double/float/binary/nchar columns, abort", pQInfo); + qError("QInfo:%p group by not supported on double/float columns, abort", pQInfo); longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } - SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true); - if (pWindowRes == NULL) { + uint64_t uid = groupIndex; // uid is always set to be 0. + SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true, uid); + if (pResultRow == NULL) { return -1; } @@ -1132,23 +1125,21 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat } if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - pWindowRes->key = malloc(varDataTLen(pData)); - varDataCopy(pWindowRes->key, pData); + pResultRow->key = malloc(varDataTLen(pData)); + varDataCopy(pResultRow->key, pData); } else { - pWindowRes->win.skey = v; - pWindowRes->win.ekey = v; + pResultRow->win.skey = v; + pResultRow->win.ekey = v; } - assert(pRuntimeEnv->windowResInfo.interval == 0); - - if (pWindowRes->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage); + if (pResultRow->pageId == -1) { + int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage); if (ret != 0) { return -1; } } - setWindowResOutputBuf(pRuntimeEnv, pWindowRes); + setResultOutputBuf(pRuntimeEnv, pResultRow); initCtxOutputBuf(pRuntimeEnv); return TSDB_CODE_SUCCESS; } @@ -1231,7 +1222,7 @@ static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { } static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SQuery* pQuery = pRuntimeEnv->pQuery; // in case of timestamp column, always generated results. @@ -1335,7 +1326,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); bool hasTimeWindow = false; - int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &win, masterScan, &hasTimeWindow); + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code continue; } @@ -1363,7 +1354,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS // null data, failed to allocate more memory buffer hasTimeWindow = false; - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &nextWin, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) { + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &nextWin, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) { break; } @@ -1379,7 +1370,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS if (groupbyColumnValue) { char *val = groupbyColumnData + bytes * offset; - int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, val, type, bytes); + int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, val, type, bytes, item->groupIndex); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code continue; } @@ -1518,7 +1509,8 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY * top/bottom values emerge, so does diff function */ if (functionId == TSDB_FUNC_TWA) { - STwaInfo *pTWAInfo = GET_RES_INFO(pCtx)->interResultBuf; + SResultRowCellInfo* pInfo = GET_RES_INFO(pCtx); + STwaInfo *pTWAInfo = (STwaInfo*) GET_ROWCELL_INTERBUF(pInfo); pTWAInfo->SKey = pQuery->window.skey; pTWAInfo->EKey = pQuery->window.ekey; } @@ -1532,7 +1524,9 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY pCtx->preAggVals.statis.max = pBlockInfo->window.ekey; } } else if (functionId == TSDB_FUNC_INTERP) { - SInterpInfoDetail *pInterpInfo = GET_RES_INFO(pCtx)->interResultBuf; + SResultRowCellInfo* pInfo = GET_RES_INFO(pCtx); + + SInterpInfoDetail *pInterpInfo = (SInterpInfoDetail *)GET_ROWCELL_INTERBUF(pInfo); pInterpInfo->type = (int8_t)pQuery->fillType; pInterpInfo->ts = pQuery->window.skey; pInterpInfo->primaryCol = (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX); @@ -1609,26 +1603,15 @@ static int32_t setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx return TSDB_CODE_SUCCESS; } -static FORCE_INLINE void setWindowResultInfo(SResultInfo *pResultInfo, SQuery *pQuery, bool isStableQuery, char* buf) { - char* p = buf; - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t size = pQuery->pSelectExpr[i].interBytes; - setResultInfoBuf(&pResultInfo[i], size, isStableQuery, p); - - p += size; - } -} - static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order) { qDebug("QInfo:%p setup runtime env", GET_QINFO_ADDR(pRuntimeEnv)); SQuery *pQuery = pRuntimeEnv->pQuery; - size_t size = pRuntimeEnv->interBufSize + pQuery->numOfOutput * sizeof(SResultInfo); - - pRuntimeEnv->resultInfo = calloc(1, size); pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutput, sizeof(SQLFunctionCtx)); + pRuntimeEnv->rowCellInfoOffset = calloc(pQuery->numOfOutput, sizeof(int32_t)); + pRuntimeEnv->pResultRow = getNewResultRow(pRuntimeEnv->pool);//calloc(1, sizeof(SResultRow)); - if (pRuntimeEnv->resultInfo == NULL || pRuntimeEnv->pCtx == NULL) { + if (pRuntimeEnv->pResultRow == NULL || pRuntimeEnv->pCtx == NULL || pRuntimeEnv->rowCellInfoOffset == NULL) { goto _clean; } @@ -1665,7 +1648,6 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order pCtx->inputType = pQuery->colList[index].type; } - assert(isValidDataType(pCtx->inputType)); pCtx->ptsOutputBuf = NULL; @@ -1674,6 +1656,8 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order pCtx->order = pQuery->order.order; pCtx->functionId = pSqlFuncMsg->functionId; + pCtx->stableQuery = pRuntimeEnv->stableQuery; + pCtx->interBufBytes = pQuery->pSelectExpr[i].interBytes; pCtx->numOfParams = pSqlFuncMsg->numOfParams; for (int32_t j = 0; j < pCtx->numOfParams; ++j) { @@ -1703,16 +1687,14 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order if (i > 0) { pRuntimeEnv->offset[i] = pRuntimeEnv->offset[i - 1] + pRuntimeEnv->pCtx[i - 1].outputBytes; + pRuntimeEnv->rowCellInfoOffset[i] = pRuntimeEnv->rowCellInfoOffset[i - 1] + sizeof(SResultRowCellInfo) + pQuery->pSelectExpr[i - 1].interBytes; } - } - char* buf = (char*) pRuntimeEnv->resultInfo + sizeof(SResultInfo) * pQuery->numOfOutput; - - // set the intermediate result output buffer - setWindowResultInfo(pRuntimeEnv->resultInfo, pQuery, pRuntimeEnv->stableQuery, buf); + } // if it is group by normal column, do not set output buffer, the output buffer is pResult - if (!pRuntimeEnv->groupbyNormalCol && !pRuntimeEnv->stableQuery) { + // fixed output query/multi-output query for normal table + if (!pRuntimeEnv->groupbyNormalCol && !pRuntimeEnv->stableQuery && !QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQuery)) { resetCtxOutputBuf(pRuntimeEnv); } @@ -1724,7 +1706,6 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order return TSDB_CODE_SUCCESS; _clean: - taosTFree(pRuntimeEnv->resultInfo); taosTFree(pRuntimeEnv->pCtx); return TSDB_CODE_QRY_OUT_OF_MEMORY; @@ -1753,17 +1734,22 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { taosTFree(pCtx->tagInfo.pTagCtxList); } - taosTFree(pRuntimeEnv->resultInfo); taosTFree(pRuntimeEnv->pCtx); } - pRuntimeEnv->pFillInfo = taosDestoryFillInfo(pRuntimeEnv->pFillInfo); + pRuntimeEnv->pFillInfo = taosDestroyFillInfo(pRuntimeEnv->pFillInfo); destroyResultBuf(pRuntimeEnv->pResultBuf); tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); pRuntimeEnv->pTSBuf = tsBufDestroy(pRuntimeEnv->pTSBuf); + taosTFree(pRuntimeEnv->keyBuf); + + taosHashCleanup(pRuntimeEnv->pResultRowHashTable); + pRuntimeEnv->pResultRowHashTable = NULL; + + pRuntimeEnv->pool = destroyResultRowPool(pRuntimeEnv->pool); } #define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED) @@ -2262,7 +2248,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo * pW TSKEY k = QUERY_IS_ASC_QUERY(pQuery)? pBlockInfo->window.skey:pBlockInfo->window.ekey; STimeWindow win = getActiveTimeWindow(pWindowResInfo, k, pQuery); - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pBlockInfo->tid, &win, masterScan, &hasTimeWindow) != + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pBlockInfo, &win, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) { // todo handle error in set result for timewindow } @@ -2648,7 +2634,7 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { } } -static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowResult *pWindowRes, bool mergeFlag) { +static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SResultRow *pWindowRes, bool mergeFlag) { SQuery * pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; @@ -2822,14 +2808,14 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) } SWindowResInfo *pWindowResInfo1 = &supporter->pTableQueryInfo[left]->windowResInfo; - SWindowResult * pWindowRes1 = getWindowResult(pWindowResInfo1, leftPos); + SResultRow * pWindowRes1 = getResultRow(pWindowResInfo1, leftPos); tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes1->pageId); char *b1 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes1, page1); TSKEY leftTimestamp = GET_INT64_VAL(b1); SWindowResInfo *pWindowResInfo2 = &supporter->pTableQueryInfo[right]->windowResInfo; - SWindowResult * pWindowRes2 = getWindowResult(pWindowResInfo2, rightPos); + SResultRow * pWindowRes2 = getResultRow(pWindowResInfo2, rightPos); tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes2->pageId); char *b2 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes2, page2); @@ -2950,7 +2936,9 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { pQuery->rec.rows += offset; } -int64_t getNumOfResultWindowRes(SQuery *pQuery, SWindowResult *pWindowRes) { +int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow *pResultRow) { + SQuery* pQuery = pRuntimeEnv->pQuery; + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functionId = pQuery->pSelectExpr[j].base.functionId; @@ -2962,7 +2950,7 @@ int64_t getNumOfResultWindowRes(SQuery *pQuery, SWindowResult *pWindowRes) { continue; } - SResultInfo *pResultInfo = &pWindowRes->resultInfo[j]; + SResultRowCellInfo *pResultInfo = getResultCell(pRuntimeEnv, pResultRow, j); assert(pResultInfo != NULL); if (pResultInfo->numOfRes > 0) { @@ -3031,18 +3019,8 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { SLoserTreeInfo *pTree = NULL; tLoserTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn); - SResultInfo *pResultInfo = calloc(pQuery->numOfOutput, sizeof(SResultInfo)); - if (pResultInfo == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - char* buf = calloc(1, pRuntimeEnv->interBufSize); - if (buf == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - setWindowResultInfo(pResultInfo, pQuery, pRuntimeEnv->stableQuery, buf); - resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo); + SResultRow* pRow = getNewResultRow(pRuntimeEnv->pool); + resetMergeResultBuf(pRuntimeEnv, pRuntimeEnv->pCtx, pRow); pQInfo->groupResInfo.groupId = getGroupResultId(pQInfo->groupIndex); @@ -3057,23 +3035,20 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { taosTFree(pTableList); taosTFree(posList); taosTFree(pTree); - taosTFree(pResultInfo); - taosTFree(buf); - longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } int32_t pos = pTree->pNode[0].index; SWindowResInfo *pWindowResInfo = &pTableList[pos]->windowResInfo; - SWindowResult *pWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]); + SResultRow *pWindowRes = getResultRow(pWindowResInfo, cs.position[pos]); tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId); char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page); TSKEY ts = GET_INT64_VAL(b); assert(ts == pWindowRes->win.skey); - int64_t num = getNumOfResultWindowRes(pQuery, pWindowRes); + int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pWindowRes); if (num <= 0) { cs.position[pos] += 1; @@ -3094,7 +3069,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { return -1; } - resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo); + resetMergeResultBuf(pRuntimeEnv, pRuntimeEnv->pCtx, pRow); } doMerge(pRuntimeEnv, ts, pWindowRes, false); @@ -3116,7 +3091,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { } } else { // current page is not needed anymore - SWindowResult *pNextWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]); + SResultRow *pNextWindowRes = getResultRow(pWindowResInfo, cs.position[pos]); if (pNextWindowRes->pageId != currentPageId) { releaseResBufPage(pRuntimeEnv->pResultBuf, page); } @@ -3133,8 +3108,6 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { taosTFree(pTree); taosTFree(pTableList); taosTFree(posList); - taosTFree(pResultInfo); - return -1; } } @@ -3151,8 +3124,8 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { taosTFree(posList); taosTFree(pTree); - taosTFree(pResultInfo); - taosTFree(buf); +// taosTFree(pResultInfo); +// taosTFree(buf); return pQInfo->groupResInfo.numOfDataPages; } @@ -3195,12 +3168,14 @@ int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupR return TSDB_CODE_SUCCESS; } -void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pResultInfo) { +void resetMergeResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx *pCtx, SResultRow *pRow) { + SQuery* pQuery = pRuntimeEnv->pQuery; + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { pCtx[k].aOutputBuf = pQuery->sdata[k]->data - pCtx[k].outputBytes; pCtx[k].size = 1; pCtx[k].startOffset = 0; - pCtx[k].resultInfo = &pResultInfo[k]; + pCtx[k].resultInfo = getResultCell(pRuntimeEnv, pRow, k); pQuery->sdata[k]->num = 0; } @@ -3237,8 +3212,8 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * pTableQueryInfo->windowResInfo.curIndex = pTableQueryInfo->windowResInfo.size - 1; } -static void disableFuncInReverseScanImpl(SQInfo* pQInfo, SWindowResInfo *pWindowResInfo, int32_t order) { - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; +static void disableFuncInReverseScanImpl(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo *pWindowResInfo, int32_t order) { + SQuery* pQuery = pRuntimeEnv->pQuery; for (int32_t i = 0; i < pWindowResInfo->size; ++i) { bool closed = getTimeWindowResStatus(pWindowResInfo, i); @@ -3246,17 +3221,18 @@ static void disableFuncInReverseScanImpl(SQInfo* pQInfo, SWindowResInfo *pWindow continue; } - SWindowResult *buf = getWindowResult(pWindowResInfo, i); + SResultRow *pRow = getResultRow(pWindowResInfo, i); // open/close the specified query for each group result for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functId = pQuery->pSelectExpr[j].base.functionId; + SResultRowCellInfo* pInfo = getResultCell(pRuntimeEnv, pRow, j); if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_ASC) || ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSDB_ORDER_DESC)) { - buf->resultInfo[j].complete = false; + pInfo->complete = false; } else if (functId != TSDB_FUNC_TS && functId != TSDB_FUNC_TAG) { - buf->resultInfo[j].complete = true; + pInfo->complete = true; } } } @@ -3270,7 +3246,7 @@ void disableFuncInReverseScan(SQInfo *pQInfo) { // group by normal columns and interval query on normal table SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { - disableFuncInReverseScanImpl(pQInfo, pWindowResInfo, order); + disableFuncInReverseScanImpl(pRuntimeEnv, pWindowResInfo, order); } else { // for simple result of table query, for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { // todo refactor int32_t functId = pQuery->pSelectExpr[j].base.functionId; @@ -3320,27 +3296,16 @@ void switchCtxOrder(SQueryRuntimeEnv *pRuntimeEnv) { } } -int32_t createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, size_t interBufSize) { - int32_t numOfCols = pQuery->numOfOutput; - - size_t size = numOfCols * sizeof(SResultInfo) + interBufSize; - pResultRow->resultInfo = calloc(1, size); - if (pResultRow->resultInfo == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - +int32_t initResultRow(SResultRow *pResultRow) { + pResultRow->pCellInfo = (SResultRowCellInfo*)((char*)pResultRow + sizeof(SResultRow)); pResultRow->pageId = -1; pResultRow->rowId = -1; - - char* buf = (char*) pResultRow->resultInfo + numOfCols * sizeof(SResultInfo); - - // set the intermediate result output buffer - setWindowResultInfo(pResultRow->resultInfo, pQuery, isSTableQuery, buf); return TSDB_CODE_SUCCESS; } void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; + SResultRow* pRow = pRuntimeEnv->pResultRow; for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; @@ -3350,8 +3315,9 @@ void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { * set the output buffer information and intermediate buffer * not all queries require the interResultBuf, such as COUNT/TAGPRJ/PRJ/TAG etc. */ - RESET_RESULT_INFO(&pRuntimeEnv->resultInfo[i]); - pCtx->resultInfo = &pRuntimeEnv->resultInfo[i]; + SResultRowCellInfo* pCellInfo = getResultCell(pRuntimeEnv, pRow, i); + RESET_RESULT_INFO(pCellInfo); + pCtx->resultInfo = pCellInfo; // set the timestamp output buffer for top/bottom/diff query int32_t functionId = pQuery->pSelectExpr[i].base.functionId; @@ -3400,7 +3366,7 @@ void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { int32_t functionId = pQuery->pSelectExpr[j].base.functionId; pRuntimeEnv->pCtx[j].currentStage = 0; - SResultInfo* pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + SResultRowCellInfo* pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); if (pResInfo->initialized) { continue; } @@ -3469,12 +3435,12 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SWindowResult *pResult = getWindowResult(pWindowResInfo, i); + SResultRow *pResult = getResultRow(pWindowResInfo, i); if (!pResult->closed) { continue; } - setWindowResOutputBuf(pRuntimeEnv, pResult); + setResultOutputBuf(pRuntimeEnv, pResult); for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int16_t functId = pQuery->pSelectExpr[j].base.functionId; @@ -3483,7 +3449,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { } aAggs[functId].xNextStep(&pRuntimeEnv->pCtx[j]); - SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); toContinue |= (!pResInfo->complete); } @@ -3496,7 +3462,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { } aAggs[functId].xNextStep(&pRuntimeEnv->pCtx[j]); - SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); toContinue |= (!pResInfo->complete); } @@ -3695,12 +3661,12 @@ void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { } for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SWindowResult *buf = &pWindowResInfo->pResult[i]; + SResultRow *buf = pWindowResInfo->pResult[i]; if (!isWindowResClosed(pWindowResInfo, i)) { continue; } - setWindowResOutputBuf(pRuntimeEnv, buf); + setResultOutputBuf(pRuntimeEnv, buf); for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { aAggs[pQuery->pSelectExpr[j].base.functionId].xFinalize(&pRuntimeEnv->pCtx[j]); @@ -3747,7 +3713,7 @@ static STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, void if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) { int32_t initialSize = 16; int32_t initialThreshold = 100; - int32_t code = initWindowResInfo(&pTableQueryInfo->windowResInfo, pRuntimeEnv, initialSize, initialThreshold, TSDB_DATA_TYPE_INT); + int32_t code = initWindowResInfo(&pTableQueryInfo->windowResInfo, initialSize, initialThreshold, TSDB_DATA_TYPE_INT); if (code != TSDB_CODE_SUCCESS) { return NULL; } @@ -3762,6 +3728,7 @@ void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) { return; } + tVariantDestroy(&pTableQueryInfo->tag); cleanupTimeWindowInfo(&pTableQueryInfo->windowResInfo); } @@ -3786,9 +3753,10 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { return; } - SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex, - sizeof(groupIndex), true); - if (pWindowRes == NULL) { + uint64_t uid = 0; // uid is always set to be 0 + SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex, + sizeof(groupIndex), true, uid); + if (pResultRow == NULL) { return; } @@ -3796,8 +3764,8 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { * not assign result buffer yet, add new result buffer * all group belong to one result set, and each group result has different group id so set the id to be one */ - if (pWindowRes->pageId == -1) { - if (addNewWindowResultBuf(pWindowRes, pRuntimeEnv->pResultBuf, groupIndex, pRuntimeEnv->numOfRowsPerPage) != + if (pResultRow->pageId == -1) { + if (addNewWindowResultBuf(pResultRow, pRuntimeEnv->pResultBuf, groupIndex, pRuntimeEnv->numOfRowsPerPage) != TSDB_CODE_SUCCESS) { return; } @@ -3805,11 +3773,11 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { // record the current active group id pRuntimeEnv->prevGroupId = groupIndex; - setWindowResOutputBuf(pRuntimeEnv, pWindowRes); + setResultOutputBuf(pRuntimeEnv, pResultRow); initCtxOutputBuf(pRuntimeEnv); } -void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult) { +void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult) { SQuery *pQuery = pRuntimeEnv->pQuery; // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group @@ -3825,18 +3793,14 @@ void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult } /* - * set the output buffer information and intermediate buffer + * set the output buffer information and intermediate buffer, * not all queries require the interResultBuf, such as COUNT */ - pCtx->resultInfo = &pResult->resultInfo[i]; - - // set super table query flag - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->superTableQ = pRuntimeEnv->stableQuery; + pCtx->resultInfo = getResultCell(pRuntimeEnv, pResult, i); } } -void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult) { +void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult) { SQuery *pQuery = pRuntimeEnv->pQuery; // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group @@ -3845,7 +3809,7 @@ void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult * for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; - pCtx->resultInfo = &pResult->resultInfo[i]; + pCtx->resultInfo = getResultCell(pRuntimeEnv, pResult, i); if (pCtx->resultInfo->initialized && pCtx->resultInfo->complete) { continue; } @@ -3858,12 +3822,6 @@ void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult * pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; } - /* - * set the output buffer information and intermediate buffer - * not all queries require the interResultBuf, such as COUNT - */ - pCtx->resultInfo->superTableQ = pRuntimeEnv->stableQuery; // set super table query flag - if (!pCtx->resultInfo->initialized) { aAggs[functionId].init(pCtx); } @@ -4004,7 +3962,7 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_ qDebug("QInfo:%p start to copy data from windowResInfo to query buf", pQInfo); int32_t totalSet = numOfClosedTimeWindow(pResultInfo); - SWindowResult* result = pResultInfo->pResult; + SResultRow** result = pResultInfo->pResult; if (orderType == TSDB_ORDER_ASC) { startIdx = pQInfo->groupIndex; @@ -4017,13 +3975,13 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_ SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; for (int32_t i = startIdx; (i < totalSet) && (i >= 0); i += step) { - if (result[i].numOfRows == 0) { + if (result[i]->numOfRows == 0) { pQInfo->groupIndex += 1; pGroupResInfo->rowId = 0; continue; } - int32_t numOfRowsToCopy = result[i].numOfRows - pGroupResInfo->rowId; + int32_t numOfRowsToCopy = result[i]->numOfRows - pGroupResInfo->rowId; int32_t oldOffset = pGroupResInfo->rowId; /* @@ -4038,13 +3996,13 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_ pQInfo->groupIndex += 1; } - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, result[i].pageId); + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, result[i]->pageId); for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t size = pRuntimeEnv->pCtx[j].outputBytes; char *out = pQuery->sdata[j]->data + numOfResult * size; - char *in = getPosInResultPage(pRuntimeEnv, j, &result[i], page); + char *in = getPosInResultPage(pRuntimeEnv, j, result[i], page); memcpy(out, in + oldOffset * size, size * numOfRowsToCopy); } @@ -4091,7 +4049,7 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv) { } for (int32_t i = 0; i < pRuntimeEnv->windowResInfo.size; ++i) { - SWindowResult *pResult = &pRuntimeEnv->windowResInfo.pResult[i]; + SResultRow *pResult = pRuntimeEnv->windowResInfo.pResult[i]; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functionId = pRuntimeEnv->pCtx[j].functionId; @@ -4099,7 +4057,8 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv) { continue; } - pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes)); + SResultRowCellInfo* pCell = getResultCell(pRuntimeEnv, pResult, j); + pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pCell->numOfRes)); } } } @@ -4245,16 +4204,24 @@ static void queryCostStatis(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQueryCostInfo *pSummary = &pRuntimeEnv->summary; + uint64_t hashSize = taosHashGetMemSize(pQInfo->runtimeEnv.pResultRowHashTable); + hashSize += taosHashGetMemSize(pQInfo->tableqinfoGroupInfo.map); + pSummary->hashSize = hashSize; + // add the merge time pSummary->elapsedTime += pSummary->firstStageMergeTime; + SResultRowPool* p = pQInfo->runtimeEnv.pool; + pSummary->winInfoSize = getResultRowPoolMemSize(p); + pSummary->numOfTimeWindows = getNumOfAllocatedResultRows(p); + qDebug("QInfo:%p :cost summary: elapsed time:%"PRId64" us, first merge:%"PRId64" us, total blocks:%d, " "load block statis:%d, load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, pQInfo, pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, pSummary->loadBlockStatis, pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); - qDebug("QInfo:%p :cost summary: internal size:%"PRId64"B, numOfWin:%"PRId64, pQInfo, pSummary->internalSupSize, - pSummary->numOfTimeWindows); + qDebug("QInfo:%p :cost summary: winResPool size:%.2f Kb, numOfWin:%"PRId64", tableInfoSize:%.2f Kb, hashTable:%.2f Kb", pQInfo, pSummary->winInfoSize/1024.0, + pSummary->numOfTimeWindows, pSummary->tableInfoSize/1024.0, pSummary->hashSize/1024.0); } static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { @@ -4612,7 +4579,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo } } - code = initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, 8, threshold, type); + code = initWindowResInfo(&pRuntimeEnv->windowResInfo, 8, threshold, type); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -4632,7 +4599,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo type = TSDB_DATA_TYPE_TIMESTAMP; } - code = initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, numOfResultRows, 4096, type); + code = initWindowResInfo(&pRuntimeEnv->windowResInfo, numOfResultRows, 1024, type); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -4659,7 +4626,7 @@ static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[i]); + SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[i]); if (pResInfo != NULL) { pResInfo->complete = false; } @@ -4882,7 +4849,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo); - if (isPointInterpoQuery(pQuery) || isFirstLastRowQuery(pQuery)) { + if (isPointInterpoQuery(pQuery)) { resetCtxOutputBuf(pRuntimeEnv); assert(pQuery->limit.offset == 0 && pQuery->limit.limit != 0); @@ -4912,11 +4879,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { pRuntimeEnv->pQueryHandle = NULL; } - if (isFirstLastRowQuery(pQuery)) { - assert(0); // last_row query switch to other routine to handle - } else { - pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(pQInfo->tsdb, &cond, &gp, pQInfo); - } + pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(pQInfo->tsdb, &cond, &gp, pQInfo); taosArrayDestroy(tx); taosArrayDestroy(g1); @@ -4930,10 +4893,6 @@ static void sequentialTableProcess(SQInfo *pQInfo) { assert(taosArrayGetSize(s) >= 1); setTagVal(pRuntimeEnv, taosArrayGetP(s, 0), pQInfo->tsdb); - if (isFirstLastRowQuery(pQuery)) { - assert(taosArrayGetSize(s) == 1); - } - taosArrayDestroy(s); // here we simply set the first table as current table @@ -5012,11 +4971,12 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - pWindowResInfo->pResult[i].closed = true; // enable return all results for group by normal columns + pWindowResInfo->pResult[i]->closed = true; // enable return all results for group by normal columns - SWindowResult *pResult = &pWindowResInfo->pResult[i]; + SResultRow *pResult = pWindowResInfo->pResult[i]; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes)); + SResultRowCellInfo* pCell = getResultCell(pRuntimeEnv, pResult, j); + pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pCell->numOfRes)); } } @@ -5287,7 +5247,6 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { if (pQInfo->code != TSDB_CODE_SUCCESS || IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p query killed or error occurred, code:%s, abort", pQInfo, tstrerror(pQInfo->code)); //TODO finalizeQueryResult may cause SEGSEV, since the memory may not allocated yet, add a cleanup function instead -// finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -6278,8 +6237,6 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou calResultBufSize(pQuery); for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { - assert(pExprs[col].interBytes >= pExprs[col].bytes); - // allocate additional memory for interResults that are usually larger then final results size_t size = (size_t)((pQuery->rec.capacity + 1) * pExprs[col].bytes + pExprs[col].interBytes + sizeof(tFilePage)); pQuery->sdata[col] = (tFilePage *)calloc(1, size); @@ -6305,12 +6262,18 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQInfo->tableqinfoGroupInfo.pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES); pQInfo->tableqinfoGroupInfo.numOfTables = pTableGroupInfo->numOfTables; pQInfo->tableqinfoGroupInfo.map = taosHashInit(pTableGroupInfo->numOfTables, - taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); + taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); } int tableIndex = 0; pQInfo->runtimeEnv.interBufSize = getOutputInterResultBufSize(pQuery); + pQInfo->runtimeEnv.summary.tableInfoSize += (pTableGroupInfo->numOfTables * sizeof(STableQueryInfo)); + + pQInfo->runtimeEnv.pResultRowHashTable = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + pQInfo->runtimeEnv.keyBuf = malloc(TSDB_MAX_BYTES_PER_ROW); + pQInfo->runtimeEnv.pool = initResultRowPool(getWindowResultSize(&pQInfo->runtimeEnv)); + pQInfo->pBuf = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); if (pQInfo->pBuf == NULL) { goto _cleanup; @@ -6345,7 +6308,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou for(int32_t j = 0; j < s; ++j) { STableKeyInfo* info = taosArrayGet(pa, j); - void* buf = (char*)pQInfo->pBuf + index * sizeof(STableQueryInfo); + void* buf = (char*) pQInfo->pBuf + index * sizeof(STableQueryInfo); window.skey = info->lastKey; STableQueryInfo* item = createTableQueryInfo(&pQInfo->runtimeEnv, info->pTable, window, buf); @@ -6410,12 +6373,13 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ SQuery *pQuery = pQInfo->runtimeEnv.pQuery; STSBuf *pTSBuf = NULL; - if (pQueryMsg->tsLen > 0) { // open new file to save the result + if (pQueryMsg->tsLen > 0) { // open new file to save the result char *tsBlock = (char *) pQueryMsg + pQueryMsg->tsOffset; pTSBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder, vgId); tsBufResetPos(pTSBuf); bool ret = tsBufNextPos(pTSBuf); + UNUSED(ret); } diff --git a/src/query/src/qFill.c b/src/query/src/qFill.c index f186726c0120f2e2e34580fec0da00c2083e5da9..99fa3a8e0ffc9b53ae370d4e5860bf16a8988367 100644 --- a/src/query/src/qFill.c +++ b/src/query/src/qFill.c @@ -91,7 +91,7 @@ void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp) { pFillInfo->numOfTotal = 0; } -void* taosDestoryFillInfo(SFillInfo* pFillInfo) { +void* taosDestroyFillInfo(SFillInfo* pFillInfo) { if (pFillInfo == NULL) { return NULL; } diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index b3e97459d36c68d48c04447b159473bd179a5dbb..edb2ca687f476faf01f1a464f80957bd3916407f 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -407,14 +407,14 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { } if (pResultBuf->file != NULL) { - qDebug("QInfo:%p res output buffer closed, total:%" PRId64 " bytes, inmem size:%dbytes, file size:%"PRId64" bytes", - pResultBuf->handle, pResultBuf->totalBufSize, listNEles(pResultBuf->lruList) * pResultBuf->pageSize, - pResultBuf->fileSize); + qDebug("QInfo:%p res output buffer closed, total:%.2f Kb, inmem size:%.2f Kb, file size:%.2f", + pResultBuf->handle, pResultBuf->totalBufSize/1024.0, listNEles(pResultBuf->lruList) * pResultBuf->pageSize / 1024.0, + pResultBuf->fileSize/1024.0); fclose(pResultBuf->file); } else { - qDebug("QInfo:%p res output buffer closed, total:%" PRId64 " bytes, no file created", pResultBuf->handle, - pResultBuf->totalBufSize); + qDebug("QInfo:%p res output buffer closed, total:%.2f Kb, no file created", pResultBuf->handle, + pResultBuf->totalBufSize/1024.0); } unlink(pResultBuf->path); diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index 529da78d9efd9644edf3d91c94bd8222c01531c8..df9ea1db0367f18cf8d436636dc72d2ec67c4648 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -156,7 +156,8 @@ void* tsBufDestroy(STSBuf* pTSBuf) { } else { // tscDebug("tsBuf %p destroyed, tmp file:%s, remains", pTSBuf, pTSBuf->path); } - + + tVariantDestroy(&pTSBuf->block.tag); free(pTSBuf); return NULL; } @@ -218,6 +219,15 @@ static void shrinkBuffer(STSList* ptsData) { } } +static int32_t getTagAreaLength(tVariant* pa) { + int32_t t = sizeof(pa->nLen) * 2 + sizeof(pa->nType); + if (pa->nType != TSDB_DATA_TYPE_NULL) { + t += pa->nLen; + } + + return t; +} + static void writeDataToDisk(STSBuf* pTSBuf) { if (pTSBuf->tsData.len == 0) { return; @@ -243,19 +253,27 @@ static void writeDataToDisk(STSBuf* pTSBuf) { */ int32_t metaLen = 0; metaLen += (int32_t)fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f); - metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + int32_t trueLen = pBlock->tag.nLen; if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { + metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f); } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { - metaLen += (int32_t)fwrite(&pBlock->tag.i64Key, 1, sizeof(int64_t), pTSBuf->f); + metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + metaLen += (int32_t)fwrite(&pBlock->tag.i64Key, 1, (size_t) pBlock->tag.nLen, pTSBuf->f); + } else { + trueLen = 0; + metaLen += (int32_t)fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); } fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); - + + metaLen += (int32_t) fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + assert(metaLen == getTagAreaLength(&pBlock->tag)); + int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; pTSBuf->fileSize += blockSize; @@ -284,23 +302,28 @@ static void expandBuffer(STSList* ptsData, int32_t inputSize) { STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { STSBlock* pBlock = &pTSBuf->block; - + // clear the memory buffer - void* tmp = pBlock->payload; - memset(pBlock, 0, sizeof(STSBlock)); - pBlock->payload = tmp; - + pBlock->compLen = 0; + pBlock->padding = 0; + pBlock->numOfElem = 0; + + int32_t offset = -1; + if (order == TSDB_ORDER_DESC) { /* * set the right position for the reversed traverse, the reversed traverse is started from * the end of each comp data block */ - int32_t ret = fseek(pTSBuf->f, -(int32_t)(sizeof(pBlock->padding)), SEEK_CUR); - size_t sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f); + int32_t prev = -(int32_t) (sizeof(pBlock->padding) + sizeof(pBlock->tag.nLen)); + int32_t ret = fseek(pTSBuf->f, prev, SEEK_CUR); + size_t sz = fread(&pBlock->padding, 1, sizeof(pBlock->padding), pTSBuf->f); + sz = fread(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); UNUSED(sz); - + pBlock->compLen = pBlock->padding; - int32_t offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag); + + offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag); ret = fseek(pTSBuf->f, -offset, SEEK_CUR); UNUSED(ret); } @@ -319,7 +342,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f); } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { - sz = fread(&pBlock->tag.i64Key, sizeof(int64_t), 1, pTSBuf->f); + sz = fread(&pBlock->tag.i64Key, (size_t) pBlock->tag.nLen, 1, pTSBuf->f); } sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); @@ -327,8 +350,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { sz = fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); UNUSED(sz); sz = fread(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); - UNUSED(sz); - + if (decomp) { pTSBuf->tsData.len = tsDecompressTimestamp(pBlock->payload, pBlock->compLen, pBlock->numOfElem, pTSBuf->tsData.rawBuf, @@ -337,11 +359,20 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { // read the comp length at the length of comp block sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f); + assert(pBlock->padding == pBlock->compLen); + + int32_t n = 0; + sz = fread(&n, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); + if (pBlock->tag.nType == TSDB_DATA_TYPE_NULL) { + assert(n == 0); + } else { + assert(n == pBlock->tag.nLen); + } + UNUSED(sz); // for backwards traverse, set the start position at the end of previous block if (order == TSDB_ORDER_DESC) { - int32_t offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag); int32_t r = fseek(pTSBuf->f, -offset, SEEK_CUR); UNUSED(r); } @@ -446,7 +477,7 @@ void tsBufFlush(STSBuf* pTSBuf) { fsync(fileno(pTSBuf->f)); } -static int32_t tsBufFindVnodeIndexFromId(STSVnodeBlockInfoEx* pVnodeInfoEx, int32_t numOfVnodes, int32_t vnodeId) { +static int32_t tsBufFindVnodeById(STSVnodeBlockInfoEx* pVnodeInfoEx, int32_t numOfVnodes, int32_t vnodeId) { int32_t j = -1; for (int32_t i = 0; i < numOfVnodes; ++i) { if (pVnodeInfoEx[i].info.vnode == vnodeId) { @@ -478,7 +509,7 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int if (pTSBuf->cur.order == TSDB_ORDER_DESC) { STSBlock* pBlock = &pTSBuf->block; int32_t compBlockSize = - pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag); + pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag); int32_t ret = fseek(pTSBuf->f, -compBlockSize, SEEK_CUR); UNUSED(ret); } @@ -506,7 +537,7 @@ static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo } if (tVariantCompare(&pTSBuf->block.tag, tag) == 0) { - return i; + return (pTSBuf->cur.order == TSDB_ORDER_ASC)? i: (pBlockInfo->numOfBlocks - (i + 1)); } } @@ -575,7 +606,7 @@ static int32_t doUpdateVnodeInfo(STSBuf* pTSBuf, int64_t offset, STSVnodeBlockIn } STSVnodeBlockInfo* tsBufGetVnodeBlockInfo(STSBuf* pTSBuf, int32_t vnodeId) { - int32_t j = tsBufFindVnodeIndexFromId(pTSBuf->pData, pTSBuf->numOfVnodes, vnodeId); + int32_t j = tsBufFindVnodeById(pTSBuf->pData, pTSBuf->numOfVnodes, vnodeId); if (j == -1) { return NULL; } @@ -839,7 +870,7 @@ STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag) { return elem; } - int32_t j = tsBufFindVnodeIndexFromId(pTSBuf->pData, pTSBuf->numOfVnodes, vnodeId); + int32_t j = tsBufFindVnodeById(pTSBuf->pData, pTSBuf->numOfVnodes, vnodeId); if (j == -1) { return elem; } @@ -992,4 +1023,47 @@ void tsBufGetVnodeIdList(STSBuf* pTSBuf, int32_t* num, int32_t** vnodeId) { for(int32_t i = 0; i < size; ++i) { (*vnodeId)[i] = pTSBuf->pData[i].info.vnode; } -} \ No newline at end of file +} + +int32_t dumpFileBlockByVnodeId(STSBuf* pTSBuf, int32_t vnodeIndex, void* buf, int32_t* len, int32_t* numOfBlocks) { + assert(vnodeIndex >= 0 && vnodeIndex < pTSBuf->numOfVnodes); + STSVnodeBlockInfo *pBlockInfo = &pTSBuf->pData[vnodeIndex].info; + + *len = 0; + *numOfBlocks = 0; + + if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) { + int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f)); +// qError("%p: fseek failed: %s", pSql, tstrerror(code)); + return code; + } + + size_t s = fread(buf, 1, pBlockInfo->compLen, pTSBuf->f); + if (s != pBlockInfo->compLen) { + int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f)); +// tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code)); + return code; + } + + *len = pBlockInfo->compLen; + *numOfBlocks = pBlockInfo->numOfBlocks; + + return TSDB_CODE_SUCCESS; +} + +STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, tVariant* pTag) { + STSElem el = {.vnode = -1}; + + for (int32_t i = 0; i < pTSBuf->numOfVnodes; ++i) { + el = tsBufGetElemStartPos(pTSBuf, pTSBuf->pData[i].info.vnode, pTag); + if (el.vnode == pTSBuf->pData[i].info.vnode) { + return el; + } + } + + return el; +} + +bool tsBufIsValidElem(STSElem* pElem) { + return pElem->vnode >= 0; +} diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index ac95afffb157847269122bf14f65a3308ef5d9a2..dc968bad06f99ab58bfeb90e7dc143a450ae34b9 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -14,8 +14,9 @@ */ #include "os.h" -#include "hash.h" #include "taosmsg.h" +#include "hash.h" + #include "qExecutor.h" #include "qUtil.h" @@ -26,74 +27,36 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery) { size += pQuery->pSelectExpr[i].interBytes; } - assert(size > 0); + assert(size >= 0); return size; } -int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRuntimeEnv, int32_t size, - int32_t threshold, int16_t type) { +int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, int32_t size, int32_t threshold, int16_t type) { pWindowResInfo->capacity = size; pWindowResInfo->threshold = threshold; pWindowResInfo->type = type; - _hash_fn_t fn = taosGetDefaultHashFunction(type); - pWindowResInfo->hashList = taosHashInit(threshold, fn, true, false); - if (pWindowResInfo->hashList == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - pWindowResInfo->curIndex = -1; pWindowResInfo->size = 0; pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; - SQueryCostInfo* pSummary = &pRuntimeEnv->summary; - - // use the pointer arraylist - pWindowResInfo->pResult = calloc(threshold, sizeof(SWindowResult)); + pWindowResInfo->pResult = calloc(pWindowResInfo->capacity, POINTER_BYTES); if (pWindowResInfo->pResult == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } - pWindowResInfo->interval = pRuntimeEnv->pQuery->interval.interval; - - pSummary->internalSupSize += sizeof(SWindowResult) * threshold; - pSummary->internalSupSize += (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * pWindowResInfo->capacity; - pSummary->numOfTimeWindows = threshold; - - for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { - int32_t code = createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - return TSDB_CODE_SUCCESS; } -void destroyTimeWindowRes(SWindowResult *pWindowRes) { - if (pWindowRes == NULL) { - return; - } - - free(pWindowRes->resultInfo); -} - void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo) { if (pWindowResInfo == NULL) { return; } if (pWindowResInfo->capacity == 0) { - assert(pWindowResInfo->hashList == NULL && pWindowResInfo->pResult == NULL); + assert(/*pWindowResInfo->hashList == NULL && */pWindowResInfo->pResult == NULL); return; } - if (pWindowResInfo->pResult != NULL) { - for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { - destroyTimeWindowRes(&pWindowResInfo->pResult[i]); - } - } - - taosHashCleanup(pWindowResInfo->hashList); taosTFree(pWindowResInfo->pResult); } @@ -103,17 +66,13 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowR } for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SWindowResult *pWindowRes = &pWindowResInfo->pResult[i]; - clearTimeWindowResBuf(pRuntimeEnv, pWindowRes); + SResultRow *pWindowRes = pWindowResInfo->pResult[i]; + clearResultRow(pRuntimeEnv, pWindowRes); } pWindowResInfo->curIndex = -1; - taosHashCleanup(pWindowResInfo->hashList); pWindowResInfo->size = 0; - _hash_fn_t fn = taosGetDefaultHashFunction(pWindowResInfo->type); - pWindowResInfo->hashList = taosHashInit(pWindowResInfo->capacity, fn, true, false); - pWindowResInfo->startTime = TSKEY_INITIAL_VAL; pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; } @@ -127,13 +86,13 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo); assert(num >= 0 && num <= numOfClosed); - int16_t type = pWindowResInfo->type; - - char *key = NULL; - int16_t bytes = -1; + int16_t type = pWindowResInfo->type; + STableId* id = TSDB_TABLEID(pRuntimeEnv->pQuery->current->pTable); // uid is always set to be 0. + char *key = NULL; + int16_t bytes = -1; for (int32_t i = 0; i < num; ++i) { - SWindowResult *pResult = &pWindowResInfo->pResult[i]; + SResultRow *pResult = pWindowResInfo->pResult[i]; if (pResult->closed) { // remove the window slot from hash table // todo refactor @@ -145,7 +104,8 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { bytes = tDataTypeDesc[pWindowResInfo->type].nSize; } - taosHashRemove(pWindowResInfo->hashList, (const char *)key, bytes); + SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid); + taosHashRemove(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); } else { break; } @@ -155,19 +115,19 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { // clear all the closed windows from the window list for (int32_t k = 0; k < remain; ++k) { - copyTimeWindowResBuf(pRuntimeEnv, &pWindowResInfo->pResult[k], &pWindowResInfo->pResult[num + k]); + copyResultRow(pRuntimeEnv, pWindowResInfo->pResult[k], pWindowResInfo->pResult[num + k]); } // move the unclosed window in the front of the window list for (int32_t k = remain; k < pWindowResInfo->size; ++k) { - SWindowResult *pWindowRes = &pWindowResInfo->pResult[k]; - clearTimeWindowResBuf(pRuntimeEnv, pWindowRes); + SResultRow *pWindowRes = pWindowResInfo->pResult[k]; + clearResultRow(pRuntimeEnv, pWindowRes); } pWindowResInfo->size = remain; for (int32_t k = 0; k < pWindowResInfo->size; ++k) { - SWindowResult *pResult = &pWindowResInfo->pResult[k]; + SResultRow *pResult = pWindowResInfo->pResult[k]; if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { key = varDataVal(pResult->key); @@ -177,12 +137,15 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { bytes = tDataTypeDesc[pWindowResInfo->type].nSize; } - int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)key, bytes); + SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid); + int32_t *p = (int32_t *)taosHashGet(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); assert(p != NULL); int32_t v = (*p - num); assert(v >= 0 && v <= pWindowResInfo->size); - taosHashPut(pWindowResInfo->hashList, (char *)key, bytes, (char *)&v, sizeof(int32_t)); + + SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid); + taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), (char *)&v, sizeof(int32_t)); } pWindowResInfo->curIndex = -1; @@ -200,7 +163,7 @@ void clearClosedTimeWindow(SQueryRuntimeEnv *pRuntimeEnv) { int32_t numOfClosedTimeWindow(SWindowResInfo *pWindowResInfo) { int32_t i = 0; - while (i < pWindowResInfo->size && pWindowResInfo->pResult[i].closed) { + while (i < pWindowResInfo->size && pWindowResInfo->pResult[i]->closed) { ++i; } @@ -211,11 +174,11 @@ void closeAllTimeWindow(SWindowResInfo *pWindowResInfo) { assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size); for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - if (pWindowResInfo->pResult[i].closed) { + if (pWindowResInfo->pResult[i]->closed) { continue; } - pWindowResInfo->pResult[i].closed = true; + pWindowResInfo->pResult[i]->closed = true; } } @@ -231,19 +194,19 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_ } // get the result order - int32_t resultOrder = (pWindowResInfo->pResult[0].win.skey < pWindowResInfo->pResult[1].win.skey)? 1:-1; + int32_t resultOrder = (pWindowResInfo->pResult[0]->win.skey < pWindowResInfo->pResult[1]->win.skey)? 1:-1; if (order != resultOrder) { return; } int32_t i = 0; if (order == QUERY_ASC_FORWARD_STEP) { - TSKEY ekey = pWindowResInfo->pResult[i].win.ekey; + TSKEY ekey = pWindowResInfo->pResult[i]->win.ekey; while (i < pWindowResInfo->size && (ekey < lastKey)) { ++i; } } else if (order == QUERY_DESC_FORWARD_STEP) { - while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].win.skey > lastKey)) { + while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i]->win.skey > lastKey)) { ++i; } } @@ -254,30 +217,33 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_ } bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot) { - return (getWindowResult(pWindowResInfo, slot)->closed == true); + return (getResultRow(pWindowResInfo, slot)->closed == true); } void closeTimeWindow(SWindowResInfo *pWindowResInfo, int32_t slot) { - getWindowResult(pWindowResInfo, slot)->closed = true; + getResultRow(pWindowResInfo, slot)->closed = true; } -void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindowRes) { +void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes) { if (pWindowRes == NULL) { return; } - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId); + // the result does not put into the SDiskbasedResultBuf, ignore it. + if (pWindowRes->pageId >= 0) { + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId); - for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) { - SResultInfo *pResultInfo = &pWindowRes->resultInfo[i]; - - char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page); - size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].bytes; - memset(s, 0, size); - - RESET_RESULT_INFO(pResultInfo); + for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) { + SResultRowCellInfo *pResultInfo = &pWindowRes->pCellInfo[i]; + + char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page); + size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].bytes; + memset(s, 0, size); + + RESET_RESULT_INFO(pResultInfo); + } } - + pWindowRes->numOfRows = 0; pWindowRes->pageId = -1; pWindowRes->rowId = -1; @@ -290,7 +256,7 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow * since the attribute of "Pos" is bound to each window result when the window result is created in the * disk-based result buffer. */ -void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, const SWindowResult *src) { +void copyResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *dst, const SResultRow *src) { dst->numOfRows = src->numOfRows; dst->win = src->win; dst->closed = src->closed; @@ -298,25 +264,105 @@ void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, con int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutput; for (int32_t i = 0; i < nOutputCols; ++i) { - SResultInfo *pDst = &dst->resultInfo[i]; - SResultInfo *pSrc = &src->resultInfo[i]; + SResultRowCellInfo *pDst = getResultCell(pRuntimeEnv, dst, i); + SResultRowCellInfo *pSrc = getResultCell(pRuntimeEnv, src, i); - char *buf = pDst->interResultBuf; - memcpy(pDst, pSrc, sizeof(SResultInfo)); - pDst->interResultBuf = buf; // restore the allocated buffer +// char *buf = pDst->interResultBuf; + memcpy(pDst, pSrc, sizeof(SResultRowCellInfo) + pRuntimeEnv->pCtx[i].interBufBytes); +// pDst->interResultBuf = buf; // restore the allocated buffer // copy the result info struct - memcpy(pDst->interResultBuf, pSrc->interResultBuf, pDst->bufLen); +// memcpy(pDst->interResultBuf, pSrc->interResultBuf, pRuntimeEnv->pCtx[i].interBufBytes); // copy the output buffer data from src to dst, the position info keep unchanged tFilePage *dstpage = getResBufPage(pRuntimeEnv->pResultBuf, dst->pageId); char * dstBuf = getPosInResultPage(pRuntimeEnv, i, dst, dstpage); tFilePage *srcpage = getResBufPage(pRuntimeEnv->pResultBuf, src->pageId); - char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SWindowResult *)src, srcpage); + char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SResultRow *)src, srcpage); size_t s = pRuntimeEnv->pQuery->pSelectExpr[i].bytes; memcpy(dstBuf, srcBuf, s); } } +SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index) { + assert(index >= 0 && index < pRuntimeEnv->pQuery->numOfOutput); + return (SResultRowCellInfo*)((char*) pRow->pCellInfo + pRuntimeEnv->rowCellInfoOffset[index]); +} + +size_t getWindowResultSize(SQueryRuntimeEnv* pRuntimeEnv) { + return (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultRowCellInfo)) + pRuntimeEnv->interBufSize + sizeof(SResultRow); +} + +SResultRowPool* initResultRowPool(size_t size) { + SResultRowPool* p = calloc(1, sizeof(SResultRowPool)); + if (p == NULL) { + return NULL; + } + + p->numOfElemPerBlock = 128; + + p->elemSize = (int32_t) size; + p->blockSize = p->numOfElemPerBlock * p->elemSize; + p->position.pos = 0; + + p->pData = taosArrayInit(8, POINTER_BYTES); + return p; +} + +SResultRow* getNewResultRow(SResultRowPool* p) { + if (p == NULL) { + return NULL; + } + + void* ptr = NULL; + if (p->position.pos == 0) { + ptr = calloc(1, p->blockSize); + taosArrayPush(p->pData, &ptr); + + } else { + size_t last = taosArrayGetSize(p->pData); + + void** pBlock = taosArrayGet(p->pData, last - 1); + ptr = ((char*) (*pBlock)) + p->elemSize * p->position.pos; + } + + p->position.pos = (p->position.pos + 1)%p->numOfElemPerBlock; + initResultRow(ptr); + + return ptr; +} + +int64_t getResultRowPoolMemSize(SResultRowPool* p) { + if (p == NULL) { + return 0; + } + + return taosArrayGetSize(p->pData) * p->blockSize; +} + +int32_t getNumOfAllocatedResultRows(SResultRowPool* p) { + return (int32_t) taosArrayGetSize(p->pData) * p->numOfElemPerBlock; +} + +int32_t getNumOfUsedResultRows(SResultRowPool* p) { + return getNumOfAllocatedResultRows(p) - p->numOfElemPerBlock + p->position.pos; +} + +void* destroyResultRowPool(SResultRowPool* p) { + if (p == NULL) { + return NULL; + } + + size_t size = taosArrayGetSize(p->pData); + for(int32_t i = 0; i < size; ++i) { + void** ptr = taosArrayGet(p->pData, i); + taosTFree(*ptr); + } + + taosArrayDestroy(p->pData); + + taosTFree(p); + return NULL; +} diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp index 8cd3a9cbef0bbf268db074a935a7b25cb389944e..2892af397960df6c1730f17209a02aa227887439 100644 --- a/src/query/tests/tsBufTest.cpp +++ b/src/query/tests/tsBufTest.cpp @@ -472,13 +472,20 @@ void mergeIdenticalVnodeBufferTest() { tsBufFlush(pTSBuf2); tsBufMerge(pTSBuf1, pTSBuf2); - EXPECT_EQ(pTSBuf1->numOfVnodes, 1); + EXPECT_EQ(pTSBuf1->numOfVnodes, 2); EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num); tsBufResetPos(pTSBuf1); + + int32_t count = 0; while (tsBufNextPos(pTSBuf1)) { STSElem elem = tsBufGetElem(pTSBuf1); - EXPECT_EQ(elem.vnode, 12); + + if (count++ < numOfTags * num) { + EXPECT_EQ(elem.vnode, 12); + } else { + EXPECT_EQ(elem.vnode, 77); + } printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag->i64Key, elem.ts); } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 90e16aee53f6978be3b4b0d81172152c0b74138e..3daefc1f5f0b74da98f926d650006f2bce46e23d 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -72,8 +72,8 @@ typedef struct STableCheckInfo { STable* pTableObj; SCompInfo* pCompInfo; int32_t compSize; - int32_t numOfBlocks; // number of qualified data blocks not the original blocks - int32_t chosen; // indicate which iterator should move forward + int32_t numOfBlocks:29; // number of qualified data blocks not the original blocks + int8_t chosen:2; // indicate which iterator should move forward bool initBuf; // whether to initialize the in-memory skip list iterator or not SSkipListIterator* iter; // mem buffer skip list iterator SSkipListIterator* iiter; // imem buffer skip list iterator diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index 2a58bec8830faab4cf9909bd417a44e2e8881570..42bc136584d618e62826c307b698746ca387ff41 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -31,14 +31,16 @@ extern "C" { typedef void (*_hash_free_fn_t)(void *param); typedef struct SHashNode { - char *key; -// struct SHashNode *prev; +// char *key; struct SHashNode *next; - uint32_t hashVal; // the hash value of key, if hashVal == HASH_VALUE_IN_TRASH, this node is moved to trash + uint32_t hashVal; // the hash value of key uint32_t keyLen; // length of the key - char *data; +// char *data; } SHashNode; +#define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode)) +#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->keyLen) + typedef enum SHashLockTypeE { HASH_NO_LOCK = 0, HASH_ENTRY_LOCK = 1, @@ -175,6 +177,8 @@ void* taosHashDestroyIter(SHashMutableIterator* iter); */ int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj); +size_t taosHashGetMemSize(const SHashObj *pHashObj); + #ifdef __cplusplus } #endif diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 625d4af1ac981c5d1f5f079f5b15533a4d63ef24..22b5da1491854926d53f2f56ee402e6269fa624c 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -22,14 +22,13 @@ #define DO_FREE_HASH_NODE(_n) \ do { \ - taosTFree((_n)->data); \ taosTFree(_n); \ } while (0) #define FREE_HASH_NODE(_h, _n) \ do { \ if ((_h)->freeFp) { \ - (_h)->freeFp((_n)->data); \ + (_h)->freeFp(GET_HASH_NODE_DATA(_n)); \ } \ \ DO_FREE_HASH_NODE(_n); \ @@ -77,7 +76,7 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { static FORCE_INLINE SHashNode *doSearchInEntryList(SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) { SHashNode *pNode = pe->next; while (pNode) { - if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { + if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) { assert(pNode->hashVal == hashVal); break; } @@ -115,11 +114,15 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p * @param dsize size of actual data * @return hash node */ -static FORCE_INLINE SHashNode *doUpdateHashNode(SHashNode *pNode, SHashNode *pNewNode) { +static FORCE_INLINE SHashNode *doUpdateHashNode(SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) { assert(pNode->keyLen == pNewNode->keyLen); - SWAP(pNode->key, pNewNode->key, void *); - SWAP(pNode->data, pNewNode->data, void *); + if (prev != NULL) { + prev->next = pNewNode; + } else { + pe->next = pNewNode; + } + pNewNode->next = pNode->next; return pNewNode; } @@ -208,12 +211,14 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da assert(pNode == NULL); } + SHashNode* prev = NULL; while (pNode) { - if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { + if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) { assert(pNode->hashVal == hashVal); break; } + prev = pNode; pNode = pNode->next; } @@ -239,7 +244,10 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da } else { // not support the update operation, return error if (pHashObj->enableUpdate) { - doUpdateHashNode(pNode, pNewNode); + doUpdateHashNode(pe, prev, pNode, pNewNode); + DO_FREE_HASH_NODE(pNode); + } else { + DO_FREE_HASH_NODE(pNewNode); } if (pHashObj->type == HASH_ENTRY_LOCK) { @@ -249,7 +257,6 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da // enable resize __rd_unlock(&pHashObj->lock, pHashObj->type); - DO_FREE_HASH_NODE(pNewNode); return pHashObj->enableUpdate ? 0 : -1; } } @@ -293,13 +300,13 @@ void* taosHashGetCB(SHashObj *pHashObj, const void *key, size_t keyLen, void (*f SHashNode *pNode = doSearchInEntryList(pe, key, keyLen, hashVal); if (pNode != NULL) { if (fp != NULL) { - fp(pNode->data); + fp(GET_HASH_NODE_DATA(pNode)); } if (d != NULL) { - memcpy(d, pNode->data, dsize); + memcpy(d, GET_HASH_NODE_DATA(pNode), dsize); } else { - data = pNode->data; + data = GET_HASH_NODE_DATA(pNode); } } @@ -357,13 +364,13 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe SHashNode *pRes = NULL; // remove it - if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { + if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) { pe->num -= 1; pRes = pNode; pe->next = pNode->next; } else { while (pNode->next != NULL) { - if (((pNode->next)->keyLen == keyLen) && (memcmp((pNode->next)->key, key, keyLen) == 0)) { + if (((pNode->next)->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY((pNode->next)), key, keyLen) == 0)) { assert((pNode->next)->hashVal == hashVal); break; } @@ -392,7 +399,7 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe __rd_unlock(&pHashObj->lock, pHashObj->type); if (data != NULL && pRes != NULL) { - memcpy(data, pRes->data, dsize); + memcpy(data, GET_HASH_NODE_DATA(pRes), dsize); } if (pRes != NULL) { @@ -426,7 +433,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi // todo remove the first node SHashNode *pNode = NULL; while((pNode = pEntry->next) != NULL) { - if (fp && (!fp(param, pNode->data))) { + if (fp && (!fp(param, GET_HASH_NODE_DATA(pNode)))) { pEntry->num -= 1; atomic_sub_fetch_64(&pHashObj->size, 1); @@ -451,7 +458,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi while ((pNext = pNode->next) != NULL) { // not qualified, remove it - if (fp && (!fp(param, pNext->data))) { + if (fp && (!fp(param, GET_HASH_NODE_DATA(pNext)))) { pNode->next = pNext->next; pEntry->num -= 1; atomic_sub_fetch_64(&pHashObj->size, 1); @@ -605,7 +612,7 @@ bool taosHashIterNext(SHashMutableIterator *pIter) { } } -void *taosHashIterGet(SHashMutableIterator *iter) { return (iter == NULL) ? NULL : iter->pCur->data; } +void *taosHashIterGet(SHashMutableIterator *iter) { return (iter == NULL) ? NULL : GET_HASH_NODE_DATA(iter->pCur); } void *taosHashDestroyIter(SHashMutableIterator *iter) { if (iter == NULL) { @@ -743,21 +750,19 @@ void taosHashTableResize(SHashObj *pHashObj) { } SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { - SHashNode *pNewNode = calloc(1, sizeof(SHashNode)); + SHashNode *pNewNode = calloc(1, sizeof(SHashNode) + keyLen + dsize); if (pNewNode == NULL) { uError("failed to allocate memory, reason:%s", strerror(errno)); return NULL; } - pNewNode->data = malloc(dsize + keyLen); - memcpy(pNewNode->data, pData, dsize); - - pNewNode->key = pNewNode->data + dsize; - memcpy(pNewNode->key, key, keyLen); - pNewNode->keyLen = (uint32_t)keyLen; pNewNode->hashVal = hashVal; + + memcpy(GET_HASH_NODE_DATA(pNewNode), pData, dsize); + memcpy(GET_HASH_NODE_KEY(pNewNode), key, keyLen); + return pNewNode; } @@ -798,3 +803,11 @@ SHashNode *getNextHashNode(SHashMutableIterator *pIter) { return NULL; } + +size_t taosHashGetMemSize(const SHashObj *pHashObj) { + if (pHashObj == NULL) { + return 0; + } + + return (pHashObj->capacity * (sizeof(SHashEntry) + POINTER_BYTES)) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj); +} diff --git a/tests/script/general/parser/fill.sim b/tests/script/general/parser/fill.sim index f89c27d71fd8ba02105f502afd4df26a1870ffb5..92aa6a922cc901c88ad68f87f61e8befbed5e87c 100644 --- a/tests/script/general/parser/fill.sim +++ b/tests/script/general/parser/fill.sim @@ -116,7 +116,7 @@ if $data81 != 1 then endi # avg_with_fill -print avg_witt_constant_fill +print avg_with_constant_fill sql select avg(c1), avg(c2), avg(c3), avg(c4), avg(c5) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6, 6, 6, 6) if $rows != 9 then return -1 @@ -371,12 +371,10 @@ if $data11 != 99 then endi sql select * from $tb -#print data08 = $data08 if $data08 != NCHAR then + print expect NCHAR, actual:$data08 return -1 endi -#return -1 - # fill_into_nonarithmetic_fieds sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 20000000, 20000000, 20000000) diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim index b70fe88e815eec020b657f00a972eba74c6e2976..145c4a008f1ae26378baeabdfd5103cc807266f3 100644 --- a/tests/script/general/parser/groupby.sim +++ b/tests/script/general/parser/groupby.sim @@ -435,53 +435,53 @@ sql insert into t1 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.00 sql insert into t2 values ('2020-03-27 04:11:16.000', 1)('2020-03-27 04:11:17.000', 2) ('2020-03-27 04:11:18.000', 3) ('2020-03-27 04:11:19.000', 4) ; sql insert into t2 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.000', 2) ('2020-03-27 04:51:18.000', 3) ('2020-03-27 05:10:19.000', 4) ; -sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2; -if $rows != 40 then - return -1 -endi - -if $data01 != 1.000000000 then - return -1 -endi -if $data02 != t1 then - return -1 -endi -if $data03 != 1 then - return -1 -endi -if $data04 != 1 then - return -1 -endi - -if $data11 != 1.000000000 then - return -1 -endi -if $data12 != t1 then - return -1 -endi -if $data13 != 1 then - return -1 -endi -if $data14 != 1 then - return -1 -endi - -sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2 limit 1; -if $rows != 2 then - return -1 -endi - -if $data11 != 1.000000000 then - return -1 -endi -if $data12 != t2 then - return -1 -endi -if $data13 != 1 then - return -1 -endi -if $data14 != 2 then - return -1 -endi +#sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2; +#if $rows != 40 then +# return -1 +#endi +# +#if $data01 != 1.000000000 then +# return -1 +#endi +#if $data02 != t1 then +# return -1 +#endi +#if $data03 != 1 then +# return -1 +#endi +#if $data04 != 1 then +# return -1 +#endi +# +#if $data11 != 1.000000000 then +# return -1 +#endi +#if $data12 != t1 then +# return -1 +#endi +#if $data13 != 1 then +# return -1 +#endi +#if $data14 != 1 then +# return -1 +#endi +# +#sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2 limit 1; +#if $rows != 2 then +# return -1 +#endi +# +#if $data11 != 1.000000000 then +# return -1 +#endi +#if $data12 != t2 then +# return -1 +#endi +#if $data13 != 1 then +# return -1 +#endi +#if $data14 != 2 then +# return -1 +#endi system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/join.sim b/tests/script/general/parser/join.sim index 79b30ffe922af1e80ab242c4c95c835c9a4a790f..624c2e72c40bfee35f8b0b17646c97149eda65c1 100644 --- a/tests/script/general/parser/join.sim +++ b/tests/script/general/parser/join.sim @@ -444,6 +444,9 @@ if $rows != $val then return -1 endi +#=============================================================== +sql select first(join_tb0.c8),first(join_tb0.c9) from join_tb1 , join_tb0 where join_tb1.ts = join_tb0.ts and join_tb1.ts <= 100002 and join_tb0.c7 = true + #====================group by========================================= diff --git a/tests/script/general/parser/join_multivnode.sim b/tests/script/general/parser/join_multivnode.sim index 102d5ba93fb79ff8874660b240c26937cce6dce7..c5fcf575ae551db7dd9ae25c06e815e7889b856d 100644 --- a/tests/script/general/parser/join_multivnode.sim +++ b/tests/script/general/parser/join_multivnode.sim @@ -315,4 +315,9 @@ if $data03 != 0 then return -1 endi +sql_error select count(join_mt0.c1), first(join_mt0.c1)-last(join_mt1.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL); +sql select count(join_mt0.c1), first(join_mt0.c1)/count(*), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL); +sql select count(join_mt0.c1), first(join_mt0.c1)-last(join_mt0.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL); +sql select last(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL); + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index b8484089250cbf0b8dd61e48ddb1e044878cb56e..3dd80b8e38c8144fee218ec372e59fe262ac7a15 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -1,51 +1,51 @@ -#sleep 2000 -#run general/parser/alter.sim -#sleep 2000 -#run general/parser/alter1.sim -#sleep 2000 -#run general/parser/alter_stable.sim -#sleep 2000 -#run general/parser/auto_create_tb.sim -#sleep 2000 -#run general/parser/auto_create_tb_drop_tb.sim -#sleep 2000 -#run general/parser/col_arithmetic_operation.sim -#sleep 2000 -#run general/parser/columnValue.sim -#sleep 2000 -#run general/parser/commit.sim -#sleep 2000 -#run general/parser/create_db.sim -#sleep 2000 -#run general/parser/create_mt.sim -#sleep 2000 -#run general/parser/create_tb.sim -#sleep 2000 -#run general/parser/dbtbnameValidate.sim -#sleep 2000 -#run general/parser/fill.sim -#sleep 2000 -#run general/parser/fill_stb.sim -#sleep 2000 -##run general/parser/fill_us.sim # -#sleep 2000 -#run general/parser/first_last.sim -#sleep 2000 -#run general/parser/import_commit1.sim -#sleep 2000 -#run general/parser/import_commit2.sim -#sleep 2000 -#run general/parser/import_commit3.sim -#sleep 2000 -##run general/parser/import_file.sim -#sleep 2000 -#run general/parser/insert_tb.sim -#sleep 2000 -#run general/parser/tags_dynamically_specifiy.sim -#sleep 2000 -#run general/parser/interp.sim -#sleep 2000 -#run general/parser/lastrow.sim +sleep 2000 +run general/parser/alter.sim +sleep 2000 +run general/parser/alter1.sim +sleep 2000 +run general/parser/alter_stable.sim +sleep 2000 +run general/parser/auto_create_tb.sim +sleep 2000 +run general/parser/auto_create_tb_drop_tb.sim +sleep 2000 +run general/parser/col_arithmetic_operation.sim +sleep 2000 +run general/parser/columnValue.sim +sleep 2000 +run general/parser/commit.sim +sleep 2000 +run general/parser/create_db.sim +sleep 2000 +run general/parser/create_mt.sim +sleep 2000 +run general/parser/create_tb.sim +sleep 2000 +run general/parser/dbtbnameValidate.sim +sleep 2000 +run general/parser/fill.sim +sleep 2000 +run general/parser/fill_stb.sim +sleep 2000 +#run general/parser/fill_us.sim # +sleep 2000 +run general/parser/first_last.sim +sleep 2000 +run general/parser/import_commit1.sim +sleep 2000 +run general/parser/import_commit2.sim +sleep 2000 +run general/parser/import_commit3.sim +sleep 2000 +#run general/parser/import_file.sim +sleep 2000 +run general/parser/insert_tb.sim +sleep 2000 +run general/parser/tags_dynamically_specifiy.sim +sleep 2000 +run general/parser/interp.sim +sleep 2000 +run general/parser/lastrow.sim sleep 2000 run general/parser/limit.sim sleep 2000