diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 0b6ae064166bf4eefa8145c6b38cb3efa59e48c9..6e8559538d93f3356ca8d0fff9a936680c0fc564 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -24,6 +24,7 @@ extern "C" { * @date 2018/09/30 */ #include "os.h" +#include "tbuffer.h" #include "qextbuffer.h" #include "taosdef.h" #include "tscSecondaryMerge.h" @@ -176,8 +177,8 @@ void tscIncStreamExecutionCount(void* pStream); bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId); // get starter position of metric query condition (query on tags) in SSqlCmd.payload -SCond* tsGetSTableQueryCondPos(STagCond* pCond, uint64_t tableIndex); -void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, const char* str); +SCond* tsGetSTableQueryCond(STagCond* pCond, uint64_t uid); +void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBuffer* pBuf); void tscTagCondCopy(STagCond* dest, const STagCond* src); void tscTagCondRelease(STagCond* pCond); @@ -199,7 +200,7 @@ int32_t tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex, SQuer STableMetaInfo* tscGetMeterMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, int32_t* index); void tscClearMeterMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache); -STableMetaInfo* tscAddMeterMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, SSuperTableMeta* pMetricMeta, +STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, SArray* vgroupList, int16_t numOfTags, int16_t* tags); STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo *pQueryInfo); int32_t tscAddSubqueryInfo(SSqlCmd *pCmd); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 80c3e77eff55585bb240a4f6a817b6fda0387ca1..5c442a155b22faf6f57a286ae3738764ed9a33a2 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -43,7 +43,7 @@ struct SSqlInfo; typedef struct SSqlGroupbyExpr { int16_t tableIndex; int16_t numOfGroupCols; - SColIndexEx columnInfo[TSDB_MAX_TAGS]; // group by columns information + SColIndex columnInfo[TSDB_MAX_TAGS]; // group by columns information int16_t orderIndex; // order by column index int16_t orderType; // order by type: asc/desc } SSqlGroupbyExpr; @@ -86,7 +86,7 @@ typedef struct STableMetaInfo { /* the structure for sql function in select clause */ typedef struct SSqlExpr { char aliasName[TSDB_COL_NAME_LEN]; // as aliasName - SColIndexEx colInfo; + SColIndex colInfo; int64_t uid; // refactor use the pointer int16_t functionId; // function id in aAgg array int16_t resType; // return value type @@ -141,6 +141,7 @@ struct SLocalReducer; typedef struct SCond { uint64_t uid; + int32_t len; // length of tag query condition data char * cond; } SCond; @@ -167,8 +168,7 @@ typedef struct STagCond { SJoinInfo joinInfo; // for different table, the query condition must be seperated - SCond cond[TSDB_MAX_JOIN_TABLE_NUM]; - int16_t numOfTagCond; + SArray* pCond; } STagCond; typedef struct SParamInfo { diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 62888234019cf2fe9e77fe476b731e6918499d7a..e6722253080b56c943eb67bea5c0212ada5bda84 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -464,13 +464,15 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; } else { // normal async query continues if (pCmd->isParseFinish) { - tscTrace("%p resend data to vnode in metermeta callback since sql has been parsed completed", pSql); + tscTrace("%p re-send data to vnode in table Meta callback since sql parsed completed", pSql); STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); code = tscGetTableMeta(pSql, pTableMetaInfo); assert(code == TSDB_CODE_SUCCESS); if (pTableMetaInfo->pTableMeta) { + // todo update the submit message according to the new table meta + // 1. table uid, 2. ip address code = tscSendMsgToServer(pSql); if (code == TSDB_CODE_SUCCESS) return; } diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index c7be1b84cefa4271f8dae73c9dd3235b6597f0e0..0fa9e01c2ed570f8305f616cea3d5ae5332940b5 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -691,7 +691,7 @@ static int32_t data_req_load_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, // todo: if column in current data block are null, opt for this case static int32_t first_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { - if (pCtx->order == TSQL_SO_DESC) { + if (pCtx->order == TSDB_ORDER_DESC) { return BLK_DATA_NO_NEEDED; } @@ -704,7 +704,7 @@ static int32_t first_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, } static int32_t last_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { - if (pCtx->order == TSQL_SO_ASC) { + if (pCtx->order == TSDB_ORDER_ASC) { return BLK_DATA_NO_NEEDED; } @@ -716,7 +716,7 @@ static int32_t last_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, } static int32_t first_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { - if (pCtx->order == TSQL_SO_DESC) { + if (pCtx->order == TSDB_ORDER_DESC) { return BLK_DATA_NO_NEEDED; } @@ -732,7 +732,7 @@ static int32_t first_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY } static int32_t last_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { - if (pCtx->order == TSQL_SO_ASC) { + if (pCtx->order == TSDB_ORDER_ASC) { return BLK_DATA_NO_NEEDED; } @@ -1483,7 +1483,7 @@ static bool first_last_function_setup(SQLFunctionCtx *pCtx) { // todo opt for null block static void first_function(SQLFunctionCtx *pCtx) { - if (pCtx->order == TSQL_SO_DESC) { + if (pCtx->order == TSDB_ORDER_DESC) { return; } @@ -1513,7 +1513,7 @@ static void first_function(SQLFunctionCtx *pCtx) { } static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) { - if (pCtx->order == TSQL_SO_DESC) { + if (pCtx->order == TSDB_ORDER_DESC) { return; } @@ -1561,7 +1561,7 @@ static void first_dist_function(SQLFunctionCtx *pCtx) { * 1. data block that are not loaded * 2. scan data files in desc order */ - if (pCtx->order == TSQL_SO_DESC) { + if (pCtx->order == TSDB_ORDER_DESC) { return; } @@ -1596,7 +1596,7 @@ static void first_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { return; } - if (pCtx->order == TSQL_SO_DESC) { + if (pCtx->order == TSDB_ORDER_DESC) { return; } @@ -1654,7 +1654,7 @@ static void first_dist_func_second_merge(SQLFunctionCtx *pCtx) { * least one data in this block that is not null.(TODO opt for this case) */ static void last_function(SQLFunctionCtx *pCtx) { - if (pCtx->order == TSQL_SO_ASC) { + if (pCtx->order == TSDB_ORDER_ASC) { return; } @@ -1683,7 +1683,7 @@ static void last_function(SQLFunctionCtx *pCtx) { } static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { - if (pCtx->order == TSQL_SO_ASC) { + if (pCtx->order == TSDB_ORDER_ASC) { return; } @@ -1730,7 +1730,7 @@ static void last_dist_function(SQLFunctionCtx *pCtx) { * 1. for scan data in asc order, no need to check data * 2. for data blocks that are not loaded, no need to check data */ - if (pCtx->order == TSQL_SO_ASC) { + if (pCtx->order == TSDB_ORDER_ASC) { return; } @@ -1768,7 +1768,7 @@ static void last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { * 1. for scan data in asc order, no need to check data * 2. for data blocks that are not loaded, no need to check data */ - if (pCtx->order == TSQL_SO_ASC) { + if (pCtx->order == TSDB_ORDER_ASC) { return; } @@ -2420,10 +2420,10 @@ static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { // user specify the order of output by sort the result according to timestamp if (pCtx->param[1].i64Key == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - __compar_fn_t comparator = (pCtx->param[2].i64Key == TSQL_SO_ASC) ? resAscComparFn : resDescComparFn; + __compar_fn_t comparator = (pCtx->param[2].i64Key == TSDB_ORDER_ASC) ? resAscComparFn : resDescComparFn; qsort(tvp, pResInfo->numOfRes, POINTER_BYTES, comparator); } else if (pCtx->param[1].i64Key > PRIMARYKEY_TIMESTAMP_COL_INDEX) { - __compar_fn_t comparator = (pCtx->param[2].i64Key == TSQL_SO_ASC) ? resDataAscComparFn : resDataDescComparFn; + __compar_fn_t comparator = (pCtx->param[2].i64Key == TSDB_ORDER_ASC) ? resDataAscComparFn : resDataDescComparFn; qsort(tvp, pResInfo->numOfRes, POINTER_BYTES, comparator); } @@ -2449,7 +2449,7 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx) { int32_t orderIdx = 0; // tOrderDesc object - tOrderDescriptor *pDesc = tOrderDesCreate(&orderIdx, NUMOFCOLS, pModel, TSQL_SO_DESC); + tOrderDescriptor *pDesc = tOrderDesCreate(&orderIdx, NUMOFCOLS, pModel, TSDB_ORDER_DESC); ((SPercentileInfo *)(pResInfo->interResultBuf))->pMemBucket = tMemBucketCreate(1024, MAX_AVAILABLE_BUFFER_SIZE, pCtx->inputBytes, pCtx->inputType, pDesc); @@ -2916,7 +2916,7 @@ static void col_project_function(SQLFunctionCtx *pCtx) { INC_INIT_VAL(pCtx, pCtx->size); char *pData = GET_INPUT_CHAR(pCtx); - if (pCtx->order == TSQL_SO_ASC) { + if (pCtx->order == TSDB_ORDER_ASC) { memcpy(pCtx->aOutputBuf, pData, (size_t)pCtx->size * pCtx->inputBytes); } else { for(int32_t i = 0; i < pCtx->size; ++i) { @@ -3011,7 +3011,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - int32_t i = (pCtx->order == TSQL_SO_ASC) ? 0 : pCtx->size - 1; + int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1; TSKEY * pTimestamp = pCtx->ptsOutputBuf; @@ -3028,7 +3028,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet pCtx->param[1].i64Key = pData[i]; pCtx->param[1].nType = pCtx->inputType; - } else if ((i == 0 && pCtx->order == TSQL_SO_ASC) || (i == pCtx->size - 1 && pCtx->order == TSQL_SO_DESC)) { + } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = pData[i] - pCtx->param[1].i64Key; *pTimestamp = pCtx->ptsList[i]; @@ -3060,7 +3060,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet pCtx->param[1].i64Key = pData[i]; pCtx->param[1].nType = pCtx->inputType; - } else if ((i == 0 && pCtx->order == TSQL_SO_ASC) || (i == pCtx->size - 1 && pCtx->order == TSQL_SO_DESC)) { + } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = pData[i] - pCtx->param[1].i64Key; *pTimestamp = pCtx->ptsList[i]; @@ -3092,7 +3092,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet pCtx->param[1].dKey = pData[i]; pCtx->param[1].nType = pCtx->inputType; - } else if ((i == 0 && pCtx->order == TSQL_SO_ASC) || (i == pCtx->size - 1 && pCtx->order == TSQL_SO_DESC)) { + } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = pData[i] - pCtx->param[1].dKey; *pTimestamp = pCtx->ptsList[i]; pOutput += 1; @@ -3122,7 +3122,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet pCtx->param[1].dKey = pData[i]; pCtx->param[1].nType = pCtx->inputType; - } else if ((i == 0 && pCtx->order == TSQL_SO_ASC) || (i == pCtx->size - 1 && pCtx->order == TSQL_SO_DESC)) { + } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = pData[i] - pCtx->param[1].dKey; *pTimestamp = pCtx->ptsList[i]; @@ -3155,7 +3155,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet pCtx->param[1].i64Key = pData[i]; pCtx->param[1].nType = pCtx->inputType; - } else if ((i == 0 && pCtx->order == TSQL_SO_ASC) || (i == pCtx->size - 1 && pCtx->order == TSQL_SO_DESC)) { + } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = pData[i] - pCtx->param[1].i64Key; *pTimestamp = pCtx->ptsList[i]; pOutput += 1; @@ -3186,7 +3186,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet pCtx->param[1].i64Key = pData[i]; pCtx->param[1].nType = pCtx->inputType; - } else if ((i == 0 && pCtx->order == TSQL_SO_ASC) || (i == pCtx->size - 1 && pCtx->order == TSQL_SO_DESC)) { + } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = pData[i] - pCtx->param[1].i64Key; *pTimestamp = pCtx->ptsList[i]; @@ -3298,17 +3298,17 @@ char *arithmetic_callback_function(void *param, char *name, int32_t colId) { SArithmeticSupport *pSupport = (SArithmeticSupport *)param; SSqlFunctionExpr *pExpr = pSupport->pExpr; - int32_t colIndexInBuf = -1; + int32_t colIndex = -1; for (int32_t i = 0; i < pExpr->binExprInfo.numOfCols; ++i) { if (colId == pExpr->binExprInfo.pReqColumns[i].colId) { - colIndexInBuf = pExpr->binExprInfo.pReqColumns[i].colIdxInBuf; + colIndex = pExpr->binExprInfo.pReqColumns[i].colIndex; break; } } - assert(colIndexInBuf >= 0 && colId >= 0); - return pSupport->data[colIndexInBuf] + pSupport->offset * pSupport->elemSize[colIndexInBuf]; + assert(colIndex >= 0 && colId >= 0); + return pSupport->data[colIndex] + pSupport->offset * pSupport->elemSize[colIndex]; } static void arithmetic_function(SQLFunctionCtx *pCtx) { @@ -4327,7 +4327,7 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) { const char *input = GET_INPUT_CHAR(pCtx); // primary ts must be existed, so no need to check its existance - if (pCtx->order == TSQL_SO_ASC) { + if (pCtx->order == TSDB_ORDER_ASC) { tsBufAppend(pTSbuf, 0, pCtx->tag.i64Key, input, pCtx->size * TSDB_KEYSIZE); } else { for (int32_t i = pCtx->size - 1; i >= 0; --i) { diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 1c738ecb3dcdade6f0ff341c320febbfbeda5e77..41d5febab5cab2011fab07f6c084286004d1b0eb 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -240,7 +240,7 @@ static int32_t tscBuildMeterSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, pCmd->numOfCols = numOfCols; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - pQueryInfo->order.order = TSQL_SO_ASC; + pQueryInfo->order.order = TSDB_ORDER_ASC; tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 0, TSDB_DATA_TYPE_BINARY, "Field", TSDB_COL_NAME_LEN); rowLen += TSDB_COL_NAME_LEN; @@ -322,7 +322,7 @@ static int tscBuildMetricTagProjectionResult(SSqlObj *pSql) { STableIdInfo *pSidExt = tscGetMeterSidInfo(pSidList, j); for (int32_t k = 0; k < pQueryInfo->fieldsInfo.numOfOutputCols; ++k) { - SColIndexEx *pColIndex = &tscSqlExprGet(pQueryInfo, k)->colInfo; + SColIndex *pColIndex = &tscSqlExprGet(pQueryInfo, k)->colInfo; int16_t offsetId = pColIndex->colIdx; assert((pColIndex->flag & TSDB_COL_TAG) != 0); @@ -460,7 +460,7 @@ void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnNa pCmd->numOfCols = 1; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - pQueryInfo->order.order = TSQL_SO_ASC; + pQueryInfo->order.order = TSDB_ORDER_ASC; tscClearFieldInfo(&pQueryInfo->fieldsInfo); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 4462af7f950b069d774ff4222310fae367d9f592..630e439397b4700f72ffcc8575cd8b0d200abc0c 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -45,7 +45,7 @@ typedef struct SColumnList { SColumnIndex ids[TSDB_MAX_COLUMNS]; } SColumnList; -static SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t colIdx, int32_t tableIndex); +static SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t colIndex, int32_t tableIndex); static int32_t setShowInfo(SSqlObj* pSql, SSqlInfo* pInfo); static char* getAccountId(SSqlObj* pSql); @@ -61,7 +61,7 @@ static int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pD static void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nameLength); static void getRevisedName(char* resultFieldName, int32_t functionId, int32_t maxLen, char* columnName); -static int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprItem* pItem, bool finalResult); +static int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExprItem* pItem, bool finalResult); static int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pIdList, int16_t bytes, int8_t type, char* fieldName, SSqlExpr* pSqlExpr); static int32_t changeFunctionID(int32_t optr, int16_t* functionId); @@ -116,9 +116,7 @@ static int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSql static int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo); static int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo); static int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index); - -static int32_t convertSyntaxTreeToExprTree(tExprNode **pExpr, tSQLExpr* pAst, int32_t* num, - SColIndexEx** pColIndex, SSqlExprInfo* pExprInfo); +static int32_t exprTreeFromSqlExpr(tExprNode **pExpr, tSQLExpr* pSqlExpr, SSqlExprInfo* pExprInfo, SQueryInfo* pQueryInfo, SArray* pCols); /* * Used during parsing query sql. Since the query sql usually small in length, error position @@ -136,7 +134,7 @@ static int32_t tscQueryOnlyMetricTags(SQueryInfo* pQueryInfo, bool* queryOnMetri SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId != TSDB_FUNC_TAGPRJ && - !(pExpr->functionId == TSDB_FUNC_COUNT && pExpr->colInfo.colIdx == TSDB_TBNAME_COLUMN_INDEX)) { + !(pExpr->functionId == TSDB_FUNC_COUNT && pExpr->colInfo.colIndex == TSDB_TBNAME_COLUMN_INDEX)) { *queryOnMetricTags = false; break; } @@ -1209,28 +1207,29 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel SSqlBinaryExprInfo* pBinExprInfo = &pFuncExpr->binExprInfo; tExprNode* pNode = NULL; - SColIndexEx* pColIndex = NULL; +// SArray* colList = taosArrayInit(10, sizeof(SColIndex)); - int32_t ret = convertSyntaxTreeToExprTree(&pNode, pItem->pNode, &pBinExprInfo->numOfCols, &pColIndex, &pQueryInfo->exprsInfo); + int32_t ret = exprTreeFromSqlExpr(&pNode, pItem->pNode, &pQueryInfo->exprsInfo, pQueryInfo, NULL); if (ret != TSDB_CODE_SUCCESS) { tExprTreeDestroy(&pNode, NULL); return invalidSqlErrMsg(pQueryInfo->msg, "invalid expression in select clause"); } pBinExprInfo->pBinExpr = pNode; - pBinExprInfo->pReqColumns = pColIndex; + assert(0); +// pBinExprInfo->pReqColumns = pColIndex; for(int32_t k = 0; k < pBinExprInfo->numOfCols; ++k) { - SColIndexEx* pCol = &pBinExprInfo->pReqColumns[k]; + SColIndex* pCol = &pBinExprInfo->pReqColumns[k]; for(int32_t f = 0; f < pQueryInfo->exprsInfo.numOfExprs; ++f) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, f); if (strcmp(pExpr->aliasName, pCol->name) == 0) { - pCol->colIdxInBuf = f; + pCol->colIndex = f; break; } } - assert(pCol->colIdxInBuf >= 0 && pCol->colIdxInBuf < pQueryInfo->exprsInfo.numOfExprs); + assert(pCol->colIndex >= 0 && pCol->colIndex < pQueryInfo->exprsInfo.numOfExprs); tfree(pNode); } } @@ -1288,23 +1287,23 @@ int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnLi return TSDB_CODE_SUCCESS; } -SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t colIdx, int32_t tableIndex) { +SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t colIndex, int32_t tableIndex) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; int32_t numOfCols = tscGetNumOfColumns(pTableMeta); - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, colIdx); + SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, colIndex); - int16_t functionId = (int16_t)((colIdx >= numOfCols) ? TSDB_FUNC_TAGPRJ : TSDB_FUNC_PRJ); + int16_t functionId = (int16_t)((colIndex >= numOfCols) ? TSDB_FUNC_TAGPRJ : TSDB_FUNC_PRJ); if (functionId == TSDB_FUNC_TAGPRJ) { -// addRequiredTagColumn(pQueryInfo, colIdx - numOfCols, tableIndex); +// addRequiredTagColumn(pQueryInfo, colIndex - numOfCols, tableIndex); pQueryInfo->type = TSDB_QUERY_TYPE_STABLE_QUERY; } else { pQueryInfo->type = TSDB_QUERY_TYPE_PROJECTION_QUERY; } - SColumnIndex index = {tableIndex, colIdx}; + SColumnIndex index = {tableIndex, colIndex}; SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, outputIndex, functionId, &index, pSchema->type, pSchema->bytes, pSchema->bytes); @@ -1503,7 +1502,7 @@ static int32_t setExprInfoForFunctions(SQueryInfo* pQueryInfo, SSchema* pSchema, return TSDB_CODE_SUCCESS; } -int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprItem* pItem, bool finalResult) { +int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExprItem* pItem, bool finalResult) { STableMetaInfo* pTableMetaInfo = NULL; int32_t optr = pItem->pNode->nSQLOptr; @@ -1547,7 +1546,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; - pExpr = tscSqlExprInsert(pQueryInfo, colIdx, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); + pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); } else { // count the number of meters created according to the metric if (getColumnIndexByName(pToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { @@ -1562,13 +1561,13 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt } int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; - pExpr = tscSqlExprInsert(pQueryInfo, colIdx, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); + pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); } } else { // count(*) is equalled to count(primary_timestamp_key) index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; - pExpr = tscSqlExprInsert(pQueryInfo, colIdx, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); + pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); } memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName)); @@ -1646,7 +1645,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt // set the first column ts for diff query if (optr == TK_DIFF) { - colIdx += 1; + colIndex += 1; SColumnIndex indexTS = {.tableIndex = index.tableIndex, .columnIndex = 0}; SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, 0, TSDB_FUNC_TS_DUMMY, &indexTS, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); @@ -1660,7 +1659,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt return invalidSqlErrMsg(pQueryInfo->msg, msg6); } - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, colIdx, functionID, &index, resultType, resultSize, resultSize); + SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionID, &index, resultType, resultSize, resultSize); if (optr == TK_LEASTSQUARES) { /* set the leastsquares parameters */ @@ -1737,7 +1736,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt for (int32_t j = 0; j < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++j) { index.columnIndex = j; - if (setExprInfoForFunctions(pQueryInfo, pSchema, functionID, pItem->aliasName, colIdx++, &index) != 0) { + if (setExprInfoForFunctions(pQueryInfo, pSchema, functionID, pItem->aliasName, colIndex++, &index) != 0) { return TSDB_CODE_INVALID_SQL; } } @@ -1755,7 +1754,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt return invalidSqlErrMsg(pQueryInfo->msg, msg6); } - if (setExprInfoForFunctions(pQueryInfo, pSchema, functionID, pItem->aliasName, colIdx + i, &index) != 0) { + if (setExprInfoForFunctions(pQueryInfo, pSchema, functionID, pItem->aliasName, colIndex + i, &index) != 0) { return TSDB_CODE_INVALID_SQL; } } @@ -1771,7 +1770,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt for (int32_t i = 0; i < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++i) { SColumnIndex index = {.tableIndex = j, .columnIndex = i}; - if (setExprInfoForFunctions(pQueryInfo, pSchema, functionID, pItem->aliasName, colIdx + i + j, &index) != + if (setExprInfoForFunctions(pQueryInfo, pSchema, functionID, pItem->aliasName, colIndex + i + j, &index) != 0) { return TSDB_CODE_INVALID_SQL; } @@ -1852,7 +1851,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt return TSDB_CODE_INVALID_SQL; } - pExpr = tscSqlExprInsert(pQueryInfo, colIdx, functionId, &index, resultType, resultSize, resultSize); + pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionId, &index, resultType, resultSize, resultSize); addExprParams(pExpr, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double), 0); } else { tVariantDump(pVariant, val, TSDB_DATA_TYPE_BIGINT); @@ -1876,9 +1875,9 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt insertResultField(pQueryInfo, TS_COLUMN_INDEX, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS].aName, pExpr); - colIdx += 1; // the first column is ts + colIndex += 1; // the first column is ts - pExpr = tscSqlExprInsert(pQueryInfo, colIdx, functionId, &index, resultType, resultSize, resultSize); + pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionId, &index, resultType, resultSize, resultSize); addExprParams(pExpr, val, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0); } @@ -1887,7 +1886,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt SColumnList ids = getColumnList(1, 0, index.columnIndex); if (finalResult) { - insertResultField(pQueryInfo, colIdx, &ids, resultSize, resultType, pExpr->aliasName, pExpr); + insertResultField(pQueryInfo, colIndex, &ids, resultSize, resultType, pExpr->aliasName, pExpr); } else { for (int32_t i = 0; i < ids.num; ++i) { tscColumnBaseInfoInsert(pQueryInfo, &(ids.ids[i])); @@ -2275,7 +2274,7 @@ int32_t tscTansformSQLFunctionForSTableQuery(SQueryInfo* pQueryInfo) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, k); int16_t functionId = aAggs[pExpr->functionId].stableFuncId; - int32_t colIndex = pExpr->colInfo.colIdx; + int32_t colIndex = pExpr->colInfo.colIndex; SSchema* pSrcSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, colIndex); if ((functionId >= TSDB_FUNC_SUM && functionId <= TSDB_FUNC_TWA) || @@ -2286,7 +2285,7 @@ int32_t tscTansformSQLFunctionForSTableQuery(SQueryInfo* pQueryInfo) { return TSDB_CODE_INVALID_SQL; } - tscSqlExprUpdate(pQueryInfo, k, functionId, pExpr->colInfo.colIdx, TSDB_DATA_TYPE_BINARY, bytes); + tscSqlExprUpdate(pQueryInfo, k, functionId, pExpr->colInfo.colIndex, TSDB_DATA_TYPE_BINARY, bytes); // todo refactor pExpr->interResBytes = intermediateBytes; } @@ -2305,7 +2304,7 @@ void tscRestoreSQLFunctionForMetricQuery(SQueryInfo* pQueryInfo) { for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); - SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIdx); + SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIndex); // if (/*(pExpr->functionId >= TSDB_FUNC_FIRST_DST && pExpr->functionId <= TSDB_FUNC_LAST_DST) || // (pExpr->functionId >= TSDB_FUNC_SUM && pExpr->functionId <= TSDB_FUNC_MAX) || @@ -2352,7 +2351,7 @@ bool hasUnsupportFunctionsForSTableQuery(SQueryInfo* pQueryInfo) { } if (pQueryInfo->groupbyExpr.numOfGroupCols != 1 || - pQueryInfo->groupbyExpr.columnInfo[0].colIdx != TSDB_TBNAME_COLUMN_INDEX) { + pQueryInfo->groupbyExpr.columnInfo[0].colIndex != TSDB_TBNAME_COLUMN_INDEX) { invalidSqlErrMsg(pQueryInfo->msg, msg2); return true; } @@ -2403,12 +2402,12 @@ void updateTagColumnIndex(SQueryInfo* pQueryInfo, int32_t tableIndex) { */ if (pQueryInfo->groupbyExpr.numOfGroupCols > 0 && pQueryInfo->groupbyExpr.tableIndex == tableIndex) { for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) { - int32_t index = pQueryInfo->groupbyExpr.columnInfo[i].colIdx; + int32_t index = pQueryInfo->groupbyExpr.columnInfo[i].colIndex; for (int32_t j = 0; j < pTableMetaInfo->numOfTags; ++j) { int32_t tagColIndex = pTableMetaInfo->tagColumnIndex[j]; if (tagColIndex == index) { - pQueryInfo->groupbyExpr.columnInfo[i].colIdx = j; + pQueryInfo->groupbyExpr.columnInfo[i].colIndex = j; break; } } @@ -2429,8 +2428,8 @@ void updateTagColumnIndex(SQueryInfo* pQueryInfo, int32_t tableIndex) { } for (int32_t j = 0; j < pTableMetaInfo->numOfTags; ++j) { - if (pExpr->colInfo.colIdx == pTableMetaInfo->tagColumnIndex[j]) { - pExpr->colInfo.colIdx = j; + if (pExpr->colInfo.colIndex == pTableMetaInfo->tagColumnIndex[j]) { + pExpr->colInfo.colIndex = j; break; } } @@ -2529,8 +2528,8 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* } pQueryInfo->groupbyExpr.columnInfo[i] = - (SColIndexEx){.colIdx = relIndex, .flag = TSDB_COL_TAG, .colId = pSchema->colId}; // relIndex; - addRequiredTagColumn(pQueryInfo, pQueryInfo->groupbyExpr.columnInfo[i].colIdx, index.tableIndex); + (SColIndex){.colIndex = relIndex, .flag = TSDB_COL_TAG, .colId = pSchema->colId}; // relIndex; + addRequiredTagColumn(pQueryInfo, pQueryInfo->groupbyExpr.columnInfo[i].colIndex, index.tableIndex); } else { // check if the column type is valid, here only support the bool/tinyint/smallint/bigint group by if (pSchema->type > TSDB_DATA_TYPE_BINARY) { @@ -2539,8 +2538,8 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* tscColumnBaseInfoInsert(pQueryInfo, &index); pQueryInfo->groupbyExpr.columnInfo[i] = - (SColIndexEx){.colIdx = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId}; // relIndex; - pQueryInfo->groupbyExpr.orderType = TSQL_SO_ASC; + (SColIndex){.colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId}; // relIndex; + pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC; if (i == 0 && pList->nExpr > 1) { return invalidSqlErrMsg(pQueryInfo->msg, msg7); @@ -2630,10 +2629,10 @@ static int32_t doExtractColumnFilterInfo(SQueryInfo* pQueryInfo, SColumnFilterIn pColumnFilter->upperRelOptr = TSDB_RELATION_LESS; break; case TK_GT: - pColumnFilter->lowerRelOptr = TSDB_RELATION_LARGE; + pColumnFilter->lowerRelOptr = TSDB_RELATION_GREATER; break; case TK_GE: - pColumnFilter->lowerRelOptr = TSDB_RELATION_LARGE_EQUAL; + pColumnFilter->lowerRelOptr = TSDB_RELATION_GREATER_EQUAL; break; case TK_EQ: pColumnFilter->lowerRelOptr = TSDB_RELATION_EQUAL; @@ -2892,6 +2891,7 @@ static void relToString(tSQLExpr* pExpr, char** str) { } } +UNUSED_FUNC static int32_t getTagCondString(tSQLExpr* pExpr, char** str) { if (pExpr == NULL) { return TSDB_CODE_SUCCESS; @@ -3593,7 +3593,7 @@ static bool validateFilterExpr(SQueryInfo* pQueryInfo) { int32_t lowerOptr = pColFilter->lowerRelOptr; int32_t upperOptr = pColFilter->upperRelOptr; - if ((lowerOptr == TSDB_RELATION_LARGE_EQUAL || lowerOptr == TSDB_RELATION_LARGE) && + if ((lowerOptr == TSDB_RELATION_GREATER_EQUAL || lowerOptr == TSDB_RELATION_GREATER) && (upperOptr == TSDB_RELATION_LESS_EQUAL || upperOptr == TSDB_RELATION_LESS)) { continue; } @@ -3727,28 +3727,27 @@ static void doAddJoinTagsColumnsIntoTagList(SQueryInfo* pQueryInfo, SCondExpr* p static int32_t getTagQueryCondExpr(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr, tSQLExpr** pExpr) { int32_t ret = TSDB_CODE_SUCCESS; - if (pCondExpr->pTagCond != NULL) { - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - tSQLExpr* p1 = extractExprForSTable(pExpr, pQueryInfo, i); - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - - char c[TSDB_MAX_TAGS_LEN] = {0}; - char* str = c; - - if ((ret = getTagCondString(p1, &str)) != TSDB_CODE_SUCCESS) { - return ret; - } - - tsSetSTableQueryCond(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid, c); - - doCompactQueryExpr(pExpr); - tSQLExprDestroy(p1); - } + if (pCondExpr->pTagCond == NULL) { + return ret; + } + + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + tSQLExpr* p1 = extractExprForSTable(pExpr, pQueryInfo, i); + tExprNode* p = NULL; + + ret = exprTreeFromSqlExpr(&p, p1, NULL, pQueryInfo, NULL); + SBuffer buf = exprTreeToBinary(p); + + int64_t uid = tscGetMetaInfo(pQueryInfo, i)->pTableMeta->uid; + tsSetSTableQueryCond(&pQueryInfo->tagCond, uid, &buf); - pCondExpr->pTagCond = NULL; + doCompactQueryExpr(pExpr); + + tSQLExprDestroy(p1); + tExprTreeDestroy(&p, NULL); } + pCondExpr->pTagCond = NULL; return ret; } int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql) { @@ -4044,11 +4043,11 @@ int32_t parseFillClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL) { static void setDefaultOrderInfo(SQueryInfo* pQueryInfo) { /* set default timestamp order information for all queries */ - pQueryInfo->order.order = TSQL_SO_ASC; + pQueryInfo->order.order = TSDB_ORDER_ASC; STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); if (isTopBottomQuery(pQueryInfo)) { - pQueryInfo->order.order = TSQL_SO_ASC; + pQueryInfo->order.order = TSDB_ORDER_ASC; pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; } else { pQueryInfo->order.orderColId = -1; @@ -4056,7 +4055,7 @@ static void setDefaultOrderInfo(SQueryInfo* pQueryInfo) { /* for metric query, set default ascending order for group output */ if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { - pQueryInfo->groupbyExpr.orderType = TSQL_SO_ASC; + pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC; } } @@ -4112,7 +4111,7 @@ int32_t parseOrderbyClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema if (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { int32_t relTagIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - if (relTagIndex == pQueryInfo->groupbyExpr.columnInfo[0].colIdx) { + if (relTagIndex == pQueryInfo->groupbyExpr.columnInfo[0].colIndex) { orderByTags = true; } } else if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { @@ -4139,7 +4138,7 @@ int32_t parseOrderbyClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema assert(pExpr->functionId == TSDB_FUNC_TS); pExpr = tscSqlExprGet(pQueryInfo, 1); - if (pExpr->colInfo.colIdx != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + if (pExpr->colInfo.colIndex != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); } @@ -4190,7 +4189,7 @@ int32_t parseOrderbyClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema assert(pExpr->functionId == TSDB_FUNC_TS); pExpr = tscSqlExprGet(pQueryInfo, 1); - if (pExpr->colInfo.colIdx != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + if (pExpr->colInfo.colIndex != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); } @@ -4705,7 +4704,7 @@ int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL* // filter the query functions operating on "tbname" column that are not supported by normal columns. for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); - if (pExpr->colInfo.colIdx == TSDB_TBNAME_COLUMN_INDEX) { + if (pExpr->colInfo.colIndex == TSDB_TBNAME_COLUMN_INDEX) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); } } @@ -4851,10 +4850,10 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClau int32_t relIndex = index.columnIndex; - pExpr->colInfo.colIdx = relIndex; - pQueryInfo->groupbyExpr.columnInfo[0].colIdx = relIndex; + pExpr->colInfo.colIndex = relIndex; + pQueryInfo->groupbyExpr.columnInfo[0].colIndex = relIndex; - addRequiredTagColumn(pQueryInfo, pQueryInfo->groupbyExpr.columnInfo[0].colIdx, 0); + addRequiredTagColumn(pQueryInfo, pQueryInfo->groupbyExpr.columnInfo[0].colIndex, 0); } } } @@ -4867,7 +4866,7 @@ static void doLimitOutputNormalColOfGroupby(SSqlExpr* pExpr) { } void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex) { - int32_t index = pQueryInfo->groupbyExpr.columnInfo[tagIndex].colIdx; + int32_t index = pQueryInfo->groupbyExpr.columnInfo[tagIndex].colIndex; STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -4909,7 +4908,7 @@ static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) { for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId != TSDB_FUNC_TAG_DUMMY && pExpr->functionId != TSDB_FUNC_TS_DUMMY) { - SSchema* pColSchema = &pSchema[pExpr->colInfo.colIdx]; + SSchema* pColSchema = &pSchema[pExpr->colInfo.colIndex]; getResultDataInfo(pColSchema->type, pColSchema->bytes, pExpr->functionId, pExpr->param[0].i64Key, &pExpr->resType, &pExpr->resBytes, &pExpr->interResBytes, tagLength, true); } @@ -5087,16 +5086,16 @@ static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) { char* name = NULL; for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) { - SColIndexEx* pColIndex = &pQueryInfo->groupbyExpr.columnInfo[i]; + SColIndex* pColIndex = &pQueryInfo->groupbyExpr.columnInfo[i]; - int16_t colIndex = pColIndex->colIdx; - if (pColIndex->colIdx == TSDB_TBNAME_COLUMN_INDEX) { + int16_t colIndex = pColIndex->colIndex; + if (pColIndex->colIndex == TSDB_TBNAME_COLUMN_INDEX) { type = TSDB_DATA_TYPE_BINARY; bytes = TSDB_TABLE_NAME_LEN; name = TSQL_TBNAME_L; } else { - colIndex = (TSDB_COL_IS_TAG(pColIndex->flag)) ? tscGetNumOfColumns(pTableMetaInfo->pTableMeta) + pColIndex->colIdx - : pColIndex->colIdx; + colIndex = (TSDB_COL_IS_TAG(pColIndex->flag)) ? tscGetNumOfColumns(pTableMetaInfo->pTableMeta) + pColIndex->colIndex + : pColIndex->colIndex; type = pSchema[colIndex].type; bytes = pSchema[colIndex].bytes; @@ -5175,7 +5174,7 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { if (functId == TSDB_FUNC_PRJ && pExpr->colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) { bool qualified = false; for (int32_t j = 0; j < pQueryInfo->groupbyExpr.numOfGroupCols; ++j) { - SColIndexEx* pColIndex = &pQueryInfo->groupbyExpr.columnInfo[j]; + SColIndex* pColIndex = &pQueryInfo->groupbyExpr.columnInfo[j]; if (pColIndex->colId == pExpr->colInfo.colId) { qualified = true; break; @@ -5192,7 +5191,7 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } - if (functId == TSDB_FUNC_COUNT && pExpr->colInfo.colIdx == TSDB_TBNAME_COLUMN_INDEX) { + if (functId == TSDB_FUNC_COUNT && pExpr->colInfo.colIndex == TSDB_TBNAME_COLUMN_INDEX) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } } @@ -5807,37 +5806,38 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return TSDB_CODE_SUCCESS; // Does not build query message here } -static int32_t convertSyntaxTreeToExprTree(tExprNode **pExpr, tSQLExpr* pAst, int32_t* num, - SColIndexEx** pColIndex, SSqlExprInfo* pExprInfo) { +int32_t exprTreeFromSqlExpr(tExprNode **pExpr, tSQLExpr* pSqlExpr, SSqlExprInfo* pExprInfo, SQueryInfo* pQueryInfo, SArray* pCols) { tExprNode* pLeft = NULL; tExprNode* pRight= NULL; - if (pAst->pLeft != NULL) { - int32_t ret = convertSyntaxTreeToExprTree(&pLeft, pAst->pLeft, num, pColIndex, pExprInfo); + if (pSqlExpr->pLeft != NULL) { + int32_t ret = exprTreeFromSqlExpr(&pLeft, pSqlExpr->pLeft, pExprInfo, pQueryInfo, pCols); if (ret != TSDB_CODE_SUCCESS) { return ret; } } - if (pAst->pRight != NULL) { - int32_t ret = convertSyntaxTreeToExprTree(&pRight, pAst->pRight, num, pColIndex, pExprInfo); + if (pSqlExpr->pRight != NULL) { + int32_t ret = exprTreeFromSqlExpr(&pRight, pSqlExpr->pRight, pExprInfo, pQueryInfo, pCols); if (ret != TSDB_CODE_SUCCESS) { return ret; } } - if (pAst->pLeft == NULL) { - if (pAst->nSQLOptr >= TK_TINYINT && pAst->nSQLOptr <= TK_DOUBLE) { - *pExpr = calloc(1, sizeof(tExprNode) + sizeof(tVariant)); + if (pSqlExpr->pLeft == NULL) { + if (pSqlExpr->nSQLOptr >= TK_TINYINT && pSqlExpr->nSQLOptr <= TK_DOUBLE) { + *pExpr = calloc(1, sizeof(tExprNode)); (*pExpr)->nodeType = TSQL_NODE_VALUE; - (*pExpr)->pVal = (tVariant*) ((char*)(*pExpr) + sizeof(tExprNode)); - tVariantAssign((*pExpr)->pVal, &pAst->val); - } else if (pAst->nSQLOptr >= TK_COUNT && pAst->nSQLOptr <= TK_AVG_IRATE) { - *pExpr = calloc(1, sizeof(tExprNode) + sizeof(SSchemaEx)); + (*pExpr)->pVal = calloc(1, sizeof(tVariant)); + + tVariantAssign((*pExpr)->pVal, &pSqlExpr->val); + + } else if (pSqlExpr->nSQLOptr >= TK_COUNT && pSqlExpr->nSQLOptr <= TK_AVG_IRATE) { + *pExpr = calloc(1, sizeof(tExprNode)); (*pExpr)->nodeType = TSQL_NODE_COL; - (*pExpr)->pSchema = (SSchema*)((char*)(*pExpr) + sizeof(tExprNode)); - strncpy((*pExpr)->pSchema->name, pAst->operand.z, pAst->operand.n); - + (*pExpr)->pSchema = calloc(1, sizeof(SSchema)); + strncpy((*pExpr)->pSchema->name, pSqlExpr->operand.z, pSqlExpr->operand.n); + // set the input column data byte and type. for (int32_t i = 0; i < pExprInfo->numOfExprs; ++i) { if (strcmp((*pExpr)->pSchema->name, pExprInfo->pExprs[i]->aliasName) == 0) { @@ -5846,22 +5846,43 @@ static int32_t convertSyntaxTreeToExprTree(tExprNode **pExpr, tSQLExpr* pAst, in break; } } - } else { //todo return error + } else if (pSqlExpr->nSQLOptr == TK_ID) { // column name + SColumnIndex index = {0}; + int32_t ret = getColumnIndexByName(&pSqlExpr->colInfo, pQueryInfo, &index); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + + *pExpr = calloc(1, sizeof(tExprNode)); + (*pExpr)->nodeType = TSQL_NODE_COL; + (*pExpr)->pSchema = calloc(1, sizeof(SSchema)); + + STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; + SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); + *(*pExpr)->pSchema = *pSchema; + return TSDB_CODE_SUCCESS; + } else { + return TSDB_CODE_INVALID_SQL; } - *pColIndex = realloc(*pColIndex, (++(*num)) * sizeof(SColIndexEx)); - memset(&(*pColIndex)[(*num) - 1], 0, sizeof(SColIndexEx)); + if (pCols != NULL) { // record the involved columns + SColIndex colIndex = {0}; + strncpy(colIndex.name, pSqlExpr->operand.z, pSqlExpr->operand.n); + taosArrayPush(pCols, &colIndex); + } - strncpy((*pColIndex)[(*num) - 1].name, pAst->operand.z, pAst->operand.n); } else { *pExpr = (tExprNode *)calloc(1, sizeof(tExprNode)); + (*pExpr)->nodeType = TSQL_NODE_EXPR; + (*pExpr)->_node.hasPK = false; (*pExpr)->_node.pLeft = pLeft; (*pExpr)->_node.pRight = pRight; - SSQLToken t = {.type = pAst->nSQLOptr}; + + SSQLToken t = {.type = pSqlExpr->nSQLOptr}; (*pExpr)->_node.optr = getBinaryExprOptr(&t); - + assert((*pExpr)->_node.optr != 0); if ((*pExpr)->_node.optr == TSDB_BINARY_OP_DIVIDE) { diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c index 1d4ed51f32a68cb39d511f83c5ea1f47a3aeb4eb..d1ec5088d3d3d9e04e1d2b961b8bbd5a5b60290f 100644 --- a/src/client/src/tscSecondaryMerge.c +++ b/src/client/src/tscSecondaryMerge.c @@ -45,7 +45,7 @@ int32_t treeComparator(const void *pLeft, const void *pRight, void *param) { return -1; } - if (pParam->groupOrderType == TSQL_SO_DESC) { // desc + if (pParam->groupOrderType == TSDB_ORDER_DESC) { // desc return compare_d(pDesc, pParam->numOfElems, pLocalData[pLeftIdx]->rowIdx, pLocalData[pLeftIdx]->filePage.data, pParam->numOfElems, pLocalData[pRightIdx]->rowIdx, pLocalData[pRightIdx]->filePage.data); } else { @@ -652,7 +652,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); - SSchema *p1 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIdx); + SSchema *p1 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIndex); int16_t inter = 0; int16_t type = -1; @@ -990,7 +990,7 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo savePrevRecordAndSetupInterpoInfo(pLocalReducer, pQueryInfo, pInterpoInfo); } - if (pQueryInfo->order.order == TSQL_SO_ASC) { + if (pQueryInfo->order.order == TSDB_ORDER_ASC) { for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); int16_t offset = getColumnModelOffset(pLocalReducer->resColModel, i); @@ -1168,7 +1168,7 @@ bool needToMerge(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer, tFilePage } else { tOrderDescriptor *pDesc = pLocalReducer->pDesc; if (pDesc->orderIdx.numOfCols > 0) { - if (pDesc->tsOrder == TSQL_SO_ASC) { // asc + if (pDesc->tsOrder == TSDB_ORDER_ASC) { // asc // todo refactor comparator ret = compare_a(pLocalReducer->pDesc, 1, 0, pLocalReducer->prevRowOfInput, 1, 0, tmpBuffer->data); } else { // desc diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 46c7dd589be8f18a8a4f100ea0f131563765096f..0bb3cb0dc790e571c9aa1a79761dc6faa6d3798c 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -37,12 +37,25 @@ int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0}; int (*tscProcessMsgRsp[TSDB_SQL_MAX])(SSqlObj *pSql); void tscProcessActivityTimer(void *handle, void *tmrId); int tscKeepConn[TSDB_SQL_MAX] = {0}; + TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid); void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts); void tscSaveSubscriptionProgress(void* sub); static int32_t minMsgSize() { return tsRpcHeadSize + 100; } +static void tscSetDnodeIpList(SSqlObj* pSql, STableMeta* pTableMeta) { + SRpcIpSet* pIpList = &pSql->ipList; + + pIpList->numOfIps = pTableMeta->numOfVpeers; + pIpList->port = tsDnodeShellPort; + pIpList->inUse = 0; + + for(int32_t i = 0; i < pTableMeta->numOfVpeers; ++i) { + pIpList->ip[i] = pTableMeta->vpeerDesc[i].ip; + } +} + void tscPrintMgmtIp() { if (tscMgmtIpList.numOfIps <= 0) { tscError("invalid mgmt IP list:%d", tscMgmtIpList.numOfIps); @@ -178,17 +191,6 @@ int tscSendMsgToServer(SSqlObj *pSql) { } if (pSql->cmd.command < TSDB_SQL_MGMT) { - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - pSql->ipList.numOfIps = pTableMeta->numOfVpeers; - pSql->ipList.port = tsDnodeShellPort; - pSql->ipList.inUse = 0; - - for(int32_t i = 0; i < pTableMeta->numOfVpeers; ++i) { - pSql->ipList.ip[i] = pTableMeta->vpeerDesc[i].ip; - } - tscPrint("%p msg:%s is sent to server %d", pSql, taosMsg[pSql->cmd.msgType], pSql->ipList.port); memcpy(pMsg, pSql->cmd.payload + tsRpcHeadSize, pSql->cmd.payloadLen); @@ -266,19 +268,18 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { rpcFreeCont(rpcMsg->pCont); return; } else { - tscTrace("%p it shall renew table meta, code:%d", pSql, tstrerror(rpcMsg->code)); + tscWarn("%p it shall renew table meta, code:%s, retry:%d", pSql, tstrerror(rpcMsg->code), ++pSql->retry); - pSql->maxRetry = TSDB_VNODES_SUPPORT * 2; + pSql->maxRetry = TSDB_VNODES_SUPPORT * 2; // todo move away pSql->res.code = rpcMsg->code; // keep the previous error code - if (++pSql->retry > pSql->maxRetry) { - tscError("%p max retry %d reached, ", pSql, pSql->retry); - return; - } - - rpcMsg->code = tscRenewMeterMeta(pSql, pTableMetaInfo->name); - - if (pTableMetaInfo->pTableMeta) { - tscSendMsgToServer(pSql); + if (pSql->retry > pSql->maxRetry) { + tscError("%p max retry %d reached, give up", pSql, pSql->maxRetry); + } else { + rpcMsg->code = tscRenewMeterMeta(pSql, pTableMetaInfo->name); + if (pTableMetaInfo->pTableMeta) { + tscSendMsgToServer(pSql); + } + rpcFreeCont(rpcMsg->pCont); return; } @@ -286,7 +287,11 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { } } - pSql->retry = 0; + if (pRes->code == TSDB_CODE_SUCCESS) { + tscTrace("%p reset retry counter to be 0 due to success rsp, old:%d", pSql, pSql->retry); + pSql->retry = 0; + } + pRes->rspLen = 0; if (pRes->code != TSDB_CODE_QUERY_CANCELLED) { @@ -331,7 +336,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { pMsg->numOfFailedBlocks = htonl(pMsg->numOfFailedBlocks); pRes->numOfRows += pMsg->affectedRows; - tscTrace("%p cmd:%d code:%d, inserted rows:%d, rsp len:%d", pSql, pCmd->command, pRes->code, + tscTrace("%p cmd:%d code:%s, inserted rows:%d, rsp len:%d", pSql, pCmd->command, tstrerror(pRes->code), pMsg->affectedRows, pRes->rspLen); } else { tscTrace("%p cmd:%d code:%s rsp len:%d", pSql, pCmd->command, tstrerror(pRes->code), pRes->rspLen); @@ -551,8 +556,9 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // pSql->cmd.payloadLen is set during copying data into payload pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; - tscTrace("%p build submit msg, vgId:%d numOfVnodes:%d", pSql, pTableMeta->vgId, htonl(pMsgDesc->numOfVnodes)); + tscSetDnodeIpList(pSql, pTableMeta); + tscTrace("%p build submit msg, vgId:%d numOfVnodes:%d", pSql, pTableMeta->vgId, htonl(pMsgDesc->numOfVnodes)); return TSDB_CODE_SUCCESS; } @@ -645,15 +651,22 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { numOfTables = 1; + tscSetDnodeIpList(pSql, pTableMeta); pQueryMsg->head.vgId = htonl(pTableMeta->vgId); tscTrace("%p queried tables:%d, table id: %s", pSql, 1, pTableMetaInfo->name); - } else { // query on super table + } else { // query super table if (pTableMetaInfo->vnodeIndex < 0) { tscError("%p error vnodeIdx:%d", pSql, pTableMetaInfo->vnodeIndex); return -1; } - uint32_t vnodeId = 1; + pSql->ipList.numOfIps = taosArrayGetSize(pTableMetaInfo->vgroupIdList); + pSql->ipList.port = tsDnodeShellPort; + pSql->ipList.inUse = 0; + + for(int32_t i = 0; i < pSql->ipList.numOfIps; ++i) { + pSql->ipList.ip[i] = *(uint32_t*) taosArrayGet(pTableMetaInfo->vgroupIdList, i); + } #if 0 SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vnodeIndex); @@ -666,12 +679,13 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } #endif + uint32_t vnodeId = 1; tscTrace("%p query on vid:%d, number of tables:%d", pSql, vnodeId, numOfTables); pQueryMsg->head.vgId = htonl(vnodeId); numOfTables = 1; } - if (pQueryInfo->order.order == TSQL_SO_ASC) { + if (pQueryInfo->order.order == TSDB_ORDER_ASC) { pQueryMsg->window.skey = htobe64(pQueryInfo->stime); pQueryMsg->window.ekey = htobe64(pQueryInfo->etime); } else { @@ -761,14 +775,14 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId)) { - /* column id is not valid according to the cached metermeta, the meter meta is expired */ + /* column id is not valid according to the cached metermeta, the table meta is expired */ tscError("%p table schema is not matched with parsed sql", pSql); return -1; } - pSqlFuncExpr->colInfo.colId = htons(pExpr->colInfo.colId); - pSqlFuncExpr->colInfo.colIdx = htons(pExpr->colInfo.colIdx); - pSqlFuncExpr->colInfo.flag = htons(pExpr->colInfo.flag); + pSqlFuncExpr->colInfo.colId = htons(pExpr->colInfo.colId); + pSqlFuncExpr->colInfo.colIndex = htons(pExpr->colInfo.colIndex); + pSqlFuncExpr->colInfo.flag = htons(pExpr->colInfo.flag); pSqlFuncExpr->functionId = htons(pExpr->functionId); pSqlFuncExpr->numOfParams = htons(pExpr->numOfParams); @@ -816,16 +830,13 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->orderType = htons(pGroupbyExpr->orderType); for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) { - SColIndexEx *pCol = &pGroupbyExpr->columnInfo[j]; + SColIndex *pCol = &pGroupbyExpr->columnInfo[j]; *((int16_t *)pMsg) = pCol->colId; pMsg += sizeof(pCol->colId); - *((int16_t *)pMsg) += pCol->colIdx; - pMsg += sizeof(pCol->colIdx); - - *((int16_t *)pMsg) += pCol->colIdxInBuf; - pMsg += sizeof(pCol->colIdxInBuf); + *((int16_t *)pMsg) += pCol->colIndex; + pMsg += sizeof(pCol->colIndex); *((int16_t *)pMsg) += pCol->flag; pMsg += sizeof(pCol->flag); @@ -867,34 +878,19 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } // serialize tag column query condition - if (pQueryInfo->tagCond.numOfTagCond > 0) { + if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0) { STagCond* pTagCond = &pQueryInfo->tagCond; - SCond *pCond = tsGetSTableQueryCondPos(pTagCond, pTableMeta->uid); + SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->uid); if (pCond != NULL && pCond->cond != NULL) { - size_t condLen = strlen(pCond->cond) + 1; + pQueryMsg->tagCondLen = htons(pCond->len); + memcpy(pMsg, pCond->cond, pCond->len); - bool ret = taosMbsToUcs4(pCond->cond, condLen, pMsg, condLen * TSDB_NCHAR_SIZE); - if (!ret) { - tscError("%p mbs to ucs4 failed:%d", pSql, tsGetSTableQueryCondPos(pTagCond, pTableMeta->uid)); - return 0; - } - - pQueryMsg->tagCondLen = htons(condLen); - pMsg += condLen * TSDB_NCHAR_SIZE; + pMsg += pCond->len; } } // tbname in/like query expression should be sent to mgmt node - STagCond* pTagCond = &pQueryInfo->tagCond; - if (pTagCond->tbnameCond.cond != NULL) { - size_t s = strlen(pTagCond->tbnameCond.cond); - memcpy(pMsg, pTagCond->tbnameCond.cond, s); - - pQueryMsg->nameCondLen = htons(s); - pMsg += s; - } - msgLen = pMsg - pStart; tscTrace("%p msg built success,len:%d bytes", pSql, msgLen); @@ -1547,7 +1543,7 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } /** - * multi meter meta req pkg format: + * multi table meta req pkg format: * | SMgmtHead | SCMMultiTableInfoMsg | tableId0 | tableId1 | tableId2 | ...... * no used 4B **/ @@ -1592,8 +1588,10 @@ static UNUSED_FUNC int32_t tscEstimateMetricMetaMsgSize(SSqlCmd *pCmd) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); int32_t n = 0; - for (int32_t i = 0; i < pQueryInfo->tagCond.numOfTagCond; ++i) { - n += strlen(pQueryInfo->tagCond.cond[i].cond); + size_t size = taosArrayGetSize(pQueryInfo->tagCond.pCond); + for (int32_t i = 0; i < size; ++i) { + assert(0); +// n += strlen(pQueryInfo->tagCond.cond[i].cond); } int32_t tagLen = n * TSDB_NCHAR_SIZE; @@ -1604,7 +1602,7 @@ static UNUSED_FUNC int32_t tscEstimateMetricMetaMsgSize(SSqlCmd *pCmd) { int32_t joinCondLen = (TSDB_TABLE_ID_LEN + sizeof(int16_t)) * 2; int32_t elemSize = sizeof(SSuperTableMetaElemMsg) * pQueryInfo->numOfTables; - int32_t colSize = pQueryInfo->groupbyExpr.numOfGroupCols*sizeof(SColIndexEx); + int32_t colSize = pQueryInfo->groupbyExpr.numOfGroupCols*sizeof(SColIndex); int32_t len = tagLen + joinCondLen + elemSize + colSize + defaultSize; @@ -1676,13 +1674,13 @@ int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // convert to unicode before sending to mnode for metric query int32_t condLen = 0; if (pTagCond->numOfTagCond > 0) { - SCond *pCond = tsGetSTableQueryCondPos(pTagCond, uid); + SCond *pCond = tsGetSTableQueryCond(pTagCond, uid); if (pCond != NULL && pCond->cond != NULL) { condLen = strlen(pCond->cond) + 1; bool ret = taosMbsToUcs4(pCond->cond, condLen, pMsg, condLen * TSDB_NCHAR_SIZE); if (!ret) { - tscError("%p mbs to ucs4 failed:%s", pSql, tsGetSTableQueryCondPos(pTagCond, uid)); + tscError("%p mbs to ucs4 failed:%s", pSql, tsGetSTableQueryCond(pTagCond, uid)); return 0; } } @@ -1729,16 +1727,16 @@ int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pElem->groupbyTagColumnList = htonl(offset); for (int32_t j = 0; j < pQueryInfo->groupbyExpr.numOfGroupCols; ++j) { - SColIndexEx *pCol = &pQueryInfo->groupbyExpr.columnInfo[j]; - SColIndexEx *pDestCol = (SColIndexEx *)pMsg; + SColIndex *pCol = &pQueryInfo->groupbyExpr.columnInfo[j]; + SColIndex *pDestCol = (SColIndex *)pMsg; pDestCol->colIdxInBuf = 0; - pDestCol->colIdx = htons(pCol->colIdx); + pDestCol->colIndex = htons(pCol->colIndex); pDestCol->colId = htons(pDestCol->colId); pDestCol->flag = htons(pDestCol->flag); strncpy(pDestCol->name, pCol->name, tListLen(pCol->name)); - pMsg += sizeof(SColIndexEx); + pMsg += sizeof(SColIndex); } } } @@ -1899,7 +1897,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { } /** - * multi meter meta rsp pkg format: + * multi table meta rsp pkg format: * | STaosRsp | ieType | SCMMultiTableInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2 * |...... 1B 1B 4B **/ @@ -2131,7 +2129,8 @@ _error_clean: // todo opt performance for(int32_t i = 0; i < pStableVgroup->numOfDnodes; ++i) { - taosArrayPush(pInfo->vgroupIdList, &pStableVgroup->dnodeIps[i]); + int32_t ip = htonl(pStableVgroup->dnodeIps[i]); + taosArrayPush(pInfo->vgroupIdList, &ip); } return pSql->res.code; @@ -2369,7 +2368,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code); static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); if (NULL == pNew) { - tscError("%p malloc failed for new sqlobj to get meter meta", pSql); + tscError("%p malloc failed for new sqlobj to get table meta", pSql); return TSDB_CODE_CLI_OUT_OF_MEMORY; } @@ -2384,7 +2383,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf pNew->cmd.autoCreated = pSql->cmd.autoCreated; // create table if not exists if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { - tscError("%p malloc failed for payload to get meter meta", pSql); + tscError("%p malloc failed for payload to get table meta", pSql); free(pNew); return TSDB_CODE_CLI_OUT_OF_MEMORY; @@ -2466,9 +2465,10 @@ int tscRenewMeterMeta(SSqlObj *pSql, char *tableId) { * 2. if get metermeta failed, still get the metermeta */ if (pTableMetaInfo->pTableMeta == NULL || !tscQueryOnMetric(pCmd)) { + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; if (pTableMetaInfo->pTableMeta) { - tscTrace("%p update meter meta, old: numOfTags:%d, numOfCols:%d, uid:%" PRId64 ", addr:%p", pSql, - pTableMetaInfo->numOfTags, pCmd->numOfCols, pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->pTableMeta); + tscTrace("%p update table meta, old: numOfTags:%d, numOfCols:%d, uid:%" PRId64 ", addr:%p", pSql, + tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->uid, pTableMeta); } tscWaitingForCreateTable(pCmd); @@ -2497,7 +2497,6 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { } #if 0 - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { char tagstr[TSDB_MAX_TAGS_LEN + 1] = {0}; @@ -2536,7 +2535,7 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { STableMetaInfo *pMMInfo = tscGetMetaInfo(pQueryInfo, i); STableMeta *pTableMeta = taosCacheAcquireByName(tscCacheHandle, pMMInfo->name); - tscAddMeterMetaInfo(pNewQueryInfo, pMMInfo->name, pTableMeta, NULL, pMMInfo->numOfTags, pMMInfo->tagColumnIndex); + tscAddTableMetaInfo(pNewQueryInfo, pMMInfo->name, pTableMeta, NULL, pMMInfo->numOfTags, pMMInfo->tagColumnIndex); } if ((code = tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index f2a3cdcab049621b1612dea3e172aac2ffca2831..c522dbfebe9365e51c8ef8aa20e2d6243401fee4 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -417,7 +417,7 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) { *rows = pRes->tsrow; - return (pQueryInfo->order.order == TSQL_SO_DESC) ? pRes->numOfRows : -pRes->numOfRows; + return (pQueryInfo->order.order == TSDB_ORDER_DESC) ? pRes->numOfRows : -pRes->numOfRows; } static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pField) { @@ -512,14 +512,14 @@ static void **doSetResultRowData(SSqlObj *pSql) { } for(int32_t k = 0; k < sas->numOfCols; ++k) { - int32_t columnIndex = sas->pExpr->binExprInfo.pReqColumns[k].colIdxInBuf; + int32_t columnIndex = sas->pExpr->binExprInfo.pReqColumns[k].colIndex; SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, columnIndex); sas->elemSize[k] = pExpr->resBytes; sas->data[k] = (pRes->data + pRes->numOfRows* pExpr->offset) + pRes->row*pExpr->resBytes; } - tSQLBinaryExprCalcTraverse(sas->pExpr->binExprInfo.pBinExpr, 1, pRes->buffer[i], sas, TSQL_SO_ASC, getArithemicInputSrc); + tSQLBinaryExprCalcTraverse(sas->pExpr->binExprInfo.pBinExpr, 1, pRes->buffer[i], sas, TSDB_ORDER_ASC, getArithemicInputSrc); pRes->tsrow[i] = pRes->buffer[i]; free(sas); //todo optimization diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 3660c822d717569abe7964751ab2df73fa728c6a..0589a6359bde43ac73bc62f27217bc7a63683d5c 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -26,7 +26,7 @@ typedef struct SInsertSupporter { static void freeSubqueryObj(SSqlObj* pSql); static bool doCompare(int32_t order, int64_t left, int64_t right) { - if (order == TSQL_SO_ASC) { + if (order == TSDB_ORDER_ASC) { return left < right; } else { return left > right; @@ -136,8 +136,8 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSubquerySupporter* pSuppor * 2. only one element for each tag. */ if (output1->tsOrder == -1) { - output1->tsOrder = TSQL_SO_ASC; - output2->tsOrder = TSQL_SO_ASC; + output1->tsOrder = TSDB_ORDER_ASC; + output2->tsOrder = TSDB_ORDER_ASC; } tsBufFlush(output1); @@ -1005,8 +1005,8 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - int32_t numOfSubQueries = taosArrayGetSize(pTableMetaInfo->vgroupIdList); - assert(numOfSubQueries > 0); + pSql->numOfSubs = taosArrayGetSize(pTableMetaInfo->vgroupIdList); + assert(pSql->numOfSubs > 0); int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, nBufferSize); if (ret != 0) { @@ -1017,16 +1017,15 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { return pRes->code; } - pSql->pSubs = calloc(numOfSubQueries, POINTER_BYTES); - pSql->numOfSubs = numOfSubQueries; + pSql->pSubs = calloc(pSql->numOfSubs, POINTER_BYTES); - tscTrace("%p retrieved query data from %d vnode(s)", pSql, numOfSubQueries); + tscTrace("%p retrieved query data from %d vnode(s)", pSql, pSql->numOfSubs); SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); - pState->numOfTotal = numOfSubQueries; + pState->numOfTotal = pSql->numOfSubs; pRes->code = TSDB_CODE_SUCCESS; int32_t i = 0; - for (; i < numOfSubQueries; ++i) { + for (; i < pSql->numOfSubs; ++i) { SRetrieveSupport *trs = (SRetrieveSupport *)calloc(1, sizeof(SRetrieveSupport)); if (trs == NULL) { tscError("%p failed to malloc buffer for SRetrieveSupport, orderOfSub:%d, reason:%s", pSql, i, strerror(errno)); @@ -1070,22 +1069,22 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { tscTrace("%p sub:%p create subquery success. orderOfSub:%d", pSql, pNew, trs->subqueryIndex); } - if (i < numOfSubQueries) { + if (i < pSql->numOfSubs) { tscError("%p failed to prepare subquery structure and launch subqueries", pSql); pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; - tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, numOfSubQueries); + tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pSql->numOfSubs); doCleanupSubqueries(pSql, i, pState); return pRes->code; // free all allocated resource } if (pRes->code == TSDB_CODE_QUERY_CANCELLED) { - tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, numOfSubQueries); + tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pSql->numOfSubs); doCleanupSubqueries(pSql, i, pState); return pRes->code; } - for(int32_t j = 0; j < numOfSubQueries; ++j) { + for(int32_t j = 0; j < pSql->numOfSubs; ++j) { SSqlObj* pSub = pSql->pSubs[j]; SRetrieveSupport* pSupport = pSub->param; @@ -1450,7 +1449,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { tscTrace("%p sub:%p reach the max retry count,set global code:%d", pParentSql, pSql, code); atomic_val_compare_exchange_32(&pState->code, 0, code); } else { // does not reach the maximum retry count, go on - tscTrace("%p sub:%p failed code:%d, retry:%d", pParentSql, pSql, code, trsupport->numOfRetry); + tscTrace("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(code), trsupport->numOfRetry); SSqlObj *pNew = tscCreateSqlObjForSubquery(pParentSql, trsupport, pSql); if (pNew == NULL) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index dd0945ab0494c99ff9a01af97614291ea8084a7c..2385b4ad823513e536ce26ce7d98fa14db657246 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -53,7 +53,7 @@ void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* str, uint64_t uid) { const int32_t maxKeySize = TSDB_MAX_TAGS_LEN; // allowed max key size - SCond* cond = tsGetSTableQueryCondPos(pTagCond, uid); + SCond* cond = tsGetSTableQueryCond(pTagCond, uid); char join[512] = {0}; if (pTagCond->joinInfo.hasJoin) { @@ -92,28 +92,41 @@ void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* str, uint64_t uid) { free(tmp); } -SCond* tsGetSTableQueryCondPos(STagCond* pTagCond, uint64_t uid) { - for (int32_t i = 0; i < TSDB_MAX_JOIN_TABLE_NUM; ++i) { - if (uid == pTagCond->cond[i].uid) { - return &pTagCond->cond[i]; +SCond* tsGetSTableQueryCond(STagCond* pTagCond, uint64_t uid) { + if (pTagCond->pCond == NULL) { + return NULL; + } + + size_t size = taosArrayGetSize(pTagCond->pCond); + for (int32_t i = 0; i < size; ++i) { + SCond* pCond = taosArrayGet(pTagCond->pCond, i); + + if (uid == pCond->uid) { + return pCond; } } return NULL; } -// todo refactor by using SArray -void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, const char* str) { - size_t len = strlen(str); - if (len == 0) { +void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBuffer* pBuf) { + if (tbufTell(pBuf) == 0) { return; } - - SCond* pDest = &pTagCond->cond[pTagCond->numOfTagCond]; - pDest->uid = uid; - pDest->cond = strdup(str); - - pTagCond->numOfTagCond += 1; + + SCond cond = { + .uid = uid, + .len = tbufTell(pBuf), + .cond = NULL, + }; + + cond.cond = tbufGetData(pBuf, true); + + if (pTagCond->pCond == NULL) { + pTagCond->pCond = taosArrayInit(3, sizeof(SCond)); + } + + taosArrayPush(pTagCond->pCond, &cond); } bool tscQueryOnMetric(SSqlCmd* pCmd) { @@ -1185,7 +1198,7 @@ SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functi } } - pExpr->colInfo.colIdx = pColIndex->columnIndex; + pExpr->colInfo.colIndex = pColIndex->columnIndex; pExpr->resType = type; pExpr->resBytes = size; pExpr->interResBytes = interSize; @@ -1207,7 +1220,7 @@ SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functi pExpr->functionId = functionId; - pExpr->colInfo.colIdx = srcColumnIndex; + pExpr->colInfo.colIndex = srcColumnIndex; pExpr->colInfo.colId = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, srcColumnIndex)->colId; pExpr->resType = type; @@ -1642,26 +1655,46 @@ void tscTagCondCopy(STagCond* dest, const STagCond* src) { dest->tbnameCond.uid = src->tbnameCond.uid; memcpy(&dest->joinInfo, &src->joinInfo, sizeof(SJoinInfo)); - - for (int32_t i = 0; i < src->numOfTagCond; ++i) { - if (src->cond[i].cond != NULL) { - dest->cond[i].cond = strdup(src->cond[i].cond); + dest->relType = src->relType; + + if (src->pCond == NULL) { + return; + } + + size_t s = taosArrayGetSize(src->pCond); + dest->pCond = taosArrayInit(s, sizeof(SCond)); + + for (int32_t i = 0; i < s; ++i) { + SCond* pCond = taosArrayGet(src->pCond, i); + + SCond c = {0}; + c.len = pCond->len; + c.uid = pCond->uid; + + if (pCond->len > 0) { + assert(pCond->cond != NULL); + c.cond = malloc(c.len); + memcpy(c.cond, pCond->cond, c.len); } - - dest->cond[i].uid = src->cond[i].uid; + + taosArrayPush(dest->pCond, &c); } - - dest->relType = src->relType; - dest->numOfTagCond = src->numOfTagCond; } -void tscTagCondRelease(STagCond* pCond) { - free(pCond->tbnameCond.cond); - for (int32_t i = 0; i < pCond->numOfTagCond; ++i) { - free(pCond->cond[i].cond); +void tscTagCondRelease(STagCond* pTagCond) { + free(pTagCond->tbnameCond.cond); + + if (pTagCond->pCond != NULL) { + size_t s = taosArrayGetSize(pTagCond->pCond); + for (int32_t i = 0; i < s; ++i) { + SCond* p = taosArrayGet(pTagCond->pCond, i); + tfree(p->cond); + } + + taosArrayDestroy(pTagCond->pCond); } - memset(pCond, 0, sizeof(STagCond)); + memset(pTagCond, 0, sizeof(STagCond)); } void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo) { @@ -1674,11 +1707,11 @@ void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo) { if (TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { SSchema* pTagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - int16_t actualTagIndex = pTableMetaInfo->tagColumnIndex[pExpr->colInfo.colIdx]; + int16_t actualTagIndex = pTableMetaInfo->tagColumnIndex[pExpr->colInfo.colIndex]; pColInfo[i].type = (actualTagIndex != -1) ? pTagSchema[actualTagIndex].type : TSDB_DATA_TYPE_BINARY; } else { - pColInfo[i].type = pSchema[pExpr->colInfo.colIdx].type; + pColInfo[i].type = pSchema[pExpr->colInfo.colIndex].type; } } } @@ -1880,8 +1913,8 @@ void tscFreeSubqueryInfo(SSqlCmd* pCmd) { tfree(pCmd->pQueryInfo); } -STableMetaInfo* tscAddMeterMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, - SSuperTableMeta* pMetricMeta, int16_t numOfTags, int16_t* tags) { +STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, + SArray* vgroupList, int16_t numOfTags, int16_t* tags) { void* pAlloc = realloc(pQueryInfo->pTableMetaInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES); if (pAlloc == NULL) { return NULL; @@ -1900,6 +1933,10 @@ STableMetaInfo* tscAddMeterMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST pTableMetaInfo->pTableMeta = pTableMeta; pTableMetaInfo->numOfTags = numOfTags; + + if (vgroupList != NULL) { + pTableMetaInfo->vgroupIdList = taosArrayClone(vgroupList); + } if (tags != NULL) { memcpy(pTableMetaInfo->tagColumnIndex, tags, sizeof(pTableMetaInfo->tagColumnIndex[0]) * numOfTags); @@ -1910,7 +1947,7 @@ STableMetaInfo* tscAddMeterMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST } STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo* pQueryInfo) { - return tscAddMeterMetaInfo(pQueryInfo, NULL, NULL, NULL, 0, NULL); + return tscAddTableMetaInfo(pQueryInfo, NULL, NULL, NULL, 0, NULL); } void doRemoveMeterMetaInfo(SQueryInfo* pQueryInfo, int32_t index, bool removeFromCache) { @@ -1961,14 +1998,14 @@ void tscResetForNextRetrieve(SSqlRes* pRes) { } SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void* param, int32_t cmd, SSqlObj* pPrevSql) { - SSqlCmd* pCmd = &pSql->cmd; - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, tableIndex); - + SSqlCmd* pCmd = &pSql->cmd; SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); if (pNew == NULL) { - tscError("%p new subquery failed, tableIndex:%d, vnodeIndex:%d", pSql, tableIndex, pTableMetaInfo->vnodeIndex); + tscError("%p new subquery failed, tableIndex:%d", pSql, tableIndex); return NULL; } + + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, tableIndex); pNew->pTscObj = pSql->pTscObj; pNew->signature = pNew; @@ -2084,12 +2121,12 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void if (pPrevSql == NULL) { STableMeta* pTableMeta = taosCacheAcquireByName(tscCacheHandle, name); - SSuperTableMeta* pMetricMeta = NULL; - if (cmd == TSDB_SQL_SELECT) { - pMetricMeta = taosCacheAcquireByName(tscCacheHandle, key); - } +// SSuperTableMeta* pMetricMeta = NULL; +// if (cmd == TSDB_SQL_SELECT) { +// pMetricMeta = taosCacheAcquireByName(tscCacheHandle, key); +// } - pFinalInfo = tscAddMeterMetaInfo(pNewQueryInfo, name, pTableMeta, pMetricMeta, pTableMetaInfo->numOfTags, + pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupIdList, pTableMetaInfo->numOfTags, pTableMetaInfo->tagColumnIndex); } else { // transfer the ownership of pTableMeta/pMetricMeta to the newly create sql object. // STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0); @@ -2097,7 +2134,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void // STableMeta* pPrevMeterMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pTableMeta); // SSuperTableMeta* pPrevMetricMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pMetricMeta); -// pFinalInfo = tscAddMeterMetaInfo(pNewQueryInfo, name, pPrevMeterMeta, pPrevMetricMeta, pTableMetaInfo->numOfTags, +// pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevMeterMeta, pPrevMetricMeta, pTableMetaInfo->numOfTags, // pTableMetaInfo->tagColumnIndex); } diff --git a/src/dnode/src/dnodeWrite.c b/src/dnode/src/dnodeWrite.c index 2e0ec5f95b1f00ae5ba60b978e7b00136bbf8efd..3c598ca360db70080e76e82ca7a8ec7f3bc6ce48 100644 --- a/src/dnode/src/dnodeWrite.c +++ b/src/dnode/src/dnodeWrite.c @@ -218,7 +218,7 @@ static void dnodeHandleIdleWorker(SWriteWorker *pWorker) { int32_t num = taosGetQueueNumber(pWorker->qset); if (num > 0) { - usleep(100); + usleep(1000); sched_yield(); } else { taosCloseQset(pWorker->qset); diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index d6a9447e3d192120a2712b46d1b2310c2732344f..de5aa1dc12725cbe1f75ee9de958586e57dd7473 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -147,10 +147,10 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); // TODO: check if below is necessary #define TSDB_RELATION_INVALID 0 #define TSDB_RELATION_LESS 1 -#define TSDB_RELATION_LARGE 2 +#define TSDB_RELATION_GREATER 2 #define TSDB_RELATION_EQUAL 3 #define TSDB_RELATION_LESS_EQUAL 4 -#define TSDB_RELATION_LARGE_EQUAL 5 +#define TSDB_RELATION_GREATER_EQUAL 5 #define TSDB_RELATION_NOT_EQUAL 6 #define TSDB_RELATION_LIKE 7 @@ -303,8 +303,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_QUERY_SET_TYPE(x, _type) ((x) |= (_type)) #define TSDB_QUERY_RESET_TYPE(x) ((x) = TSDB_QUERY_TYPE_NON_TYPE) -#define TSQL_SO_ASC 1 -#define TSQL_SO_DESC 0 +#define TSDB_ORDER_ASC 1 +#define TSDB_ORDER_DESC 2 #define TSDB_SESSIONS_PER_VNODE (300) #define TSDB_SESSIONS_PER_DNODE (TSDB_SESSIONS_PER_VNODE * TSDB_MAX_VNODES) diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index f5168a2c9eecb842a5b7e28003444cb12f7f4f32..c580bdecdc687957544bb5510a5fc37d4de577b9 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -358,7 +358,7 @@ typedef struct { int32_t vgId; } SMDDropVnodeMsg; -typedef struct SColIndexEx { +typedef struct SColIndex { int16_t colId; /* * colIdx is the index of column in latest schema of table @@ -368,11 +368,10 @@ typedef struct SColIndexEx { * colIdxInBuf is used to denote the index of column in pQuery->colList, * this value is invalid in client side, as well as in cache block of vnode either. */ - int16_t colIdx; - int16_t colIdxInBuf; + int16_t colIndex; uint16_t flag; // denote if it is a tag or not char name[TSDB_COL_NAME_LEN]; -} SColIndexEx; +} SColIndex; /* sql function msg, to describe the message to vnode about sql function * operations in select clause */ @@ -380,7 +379,7 @@ typedef struct SSqlFuncExprMsg { int16_t functionId; int16_t numOfParams; - SColIndexEx colInfo; + SColIndex colInfo; struct ArgElem { int16_t argType; int16_t argBytes; @@ -395,7 +394,7 @@ typedef struct SSqlFuncExprMsg { typedef struct SSqlBinaryExprInfo { struct tExprNode *pBinExpr; /* for binary expression */ int32_t numOfCols; /* binary expression involves the readed number of columns*/ - SColIndexEx * pReqColumns; /* source column list */ + SColIndex * pReqColumns; /* source column list */ } SSqlBinaryExprInfo; typedef struct SSqlFunctionExpr { @@ -467,7 +466,6 @@ typedef struct { int64_t slidingTime; // value for sliding window char slidingTimeUnit; // time interval type, for revisement of interval(1d) uint16_t tagCondLen; // tag length in current query - uint16_t nameCondLen; // table name in/like query expression string length int16_t numOfGroupCols; // num of group by columns int16_t orderByIdx; int16_t orderType; // used in group by xx order by xxx @@ -669,7 +667,7 @@ typedef struct { typedef struct STableMetaMsg { int32_t contLen; - char tableId[TSDB_TABLE_ID_LEN]; // table id + char tableId[TSDB_TABLE_ID_LEN]; // table id char stableId[TSDB_TABLE_ID_LEN]; // stable name if it is created according to super table uint8_t numOfTags; uint8_t precision; diff --git a/src/query/inc/qast.h b/src/query/inc/qast.h index 616a2a46afb28506d639a6ea250147fc8bbcfc08..72d6f9bf47475dbf454ed8c4eaae557f5dcb3522 100644 --- a/src/query/inc/qast.h +++ b/src/query/inc/qast.h @@ -48,7 +48,7 @@ typedef void (*__do_filter_suppl_fn_t)(void *, void *); */ typedef struct tQueryInfo { int32_t offset; // offset value in tags - int32_t colIdx; // index of column in schema + int32_t colIndex; // index of column in schema uint8_t optr; // expression operator SSchema sch; // schema of tags tVariant q; // query condition value on the specific schema, filter expression @@ -83,7 +83,7 @@ void tSQLBinaryExprToString(tExprNode *pExpr, char *dst, int32_t *len); void tExprTreeDestroy(tExprNode **pExprs, void (*fp)(void*)); -void tSQLBinaryExprTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param); +void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param); void tSQLBinaryExprCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*cb)(void *, char *, int32_t)); @@ -94,7 +94,7 @@ uint8_t getBinaryExprOptr(SSQLToken *pToken); SBuffer exprTreeToBinary(tExprNode* pExprTree); -tExprNode* exprTreeFromBinary(const void* pBuf, size_t size); +int32_t exprTreeFromBinary(const void* pBuf, size_t size, tExprNode** pExprNode); #ifdef __cplusplus } diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index 8c8c1a3a12f2392441abdd4ca673b45641cf1e22..dc5f2fcb78fba4d690ac4466643f4d2a9589021c 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -39,7 +39,7 @@ typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int typedef struct SSqlGroupbyExpr { int16_t tableIndex; int16_t numOfGroupCols; - SColIndexEx columnInfo[TSDB_MAX_TAGS]; // group by columns information + SColIndex columnInfo[TSDB_MAX_TAGS]; // group by columns information int16_t orderIndex; // order by column index int16_t orderType; // order by type: asc/desc } SSqlGroupbyExpr; @@ -63,7 +63,7 @@ typedef struct SWindowResult { typedef struct SResultRec { int64_t total; // total generated result size in rows - int64_t size; // current result set size in rows + int64_t rows; // current result set size in rows int64_t capacity; // capacity of current result output buffer // result size threshold in rows. If the result buffer is larger than this, pause query and return to client @@ -125,7 +125,7 @@ typedef struct SQuery { int8_t precision; int16_t numOfOutputCols; int16_t interpoType; - int16_t checkBufferInLoop; // check if the buffer is full during scan each block + int16_t checkBuffer; // check if the buffer is full during scan each block SLimitVal limit; int32_t rowSize; SSqlGroupbyExpr* pGroupbyExpr; diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index 585aa8a9cf8a5166e00ec23b9782bb979c671634..87b974b9bac00e392985e1f06744d5f52aab3f4e 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -466,9 +466,9 @@ item(A) ::= ids(X) cpxName(Y). { } %type sortorder {int} -sortorder(A) ::= ASC. {A = TSQL_SO_ASC; } -sortorder(A) ::= DESC. {A = TSQL_SO_DESC;} -sortorder(A) ::= . {A = TSQL_SO_ASC;} //default is descend order +sortorder(A) ::= ASC. {A = TSDB_ORDER_ASC; } +sortorder(A) ::= DESC. {A = TSDB_ORDER_DESC;} +sortorder(A) ::= . {A = TSDB_ORDER_ASC;} //default is descend order //group by clause %type groupby_opt {tVariantList*} diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index c7aa57200cff7e7fb6091e55dd4366875f6e57b7..7514f1e44c48c973f55dda3c39b469d3a9c18c3b 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -102,7 +102,7 @@ extern "C" { #define QUERY_ASC_FORWARD_STEP 1 #define QUERY_DESC_FORWARD_STEP -1 -#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSQL_SO_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) +#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) #define MAX_RETRIEVE_ROWS_IN_INTERVAL_QUERY 10000000 #define TOP_BOTTOM_QUERY_LIMIT 100 diff --git a/src/query/src/qast.c b/src/query/src/qast.c index 6674dc1abafdae5b9ccd71df7395609a551ce6ef..ffc47d2abeb1cbd9fe05652563e653a61276cc6c 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -18,6 +18,7 @@ #include "tutil.h" #include "tbuffer.h" #include "qast.h" +#include "tcompare.h" #include "qsqlparser.h" #include "qsyntaxtreefunction.h" #include "taosdef.h" @@ -79,22 +80,22 @@ static void reviseBinaryExprIfNecessary(tExprNode **pLeft, tExprNode **pRight, u * switch left and left and right hand side in expr if possible */ if ((*pLeft)->nodeType == TSQL_NODE_VALUE && (*pRight)->nodeType == TSQL_NODE_COL) { - if (*optr >= TSDB_RELATION_LARGE && *optr <= TSDB_RELATION_LARGE_EQUAL && *optr != TSDB_RELATION_EQUAL) { + if (*optr >= TSDB_RELATION_GREATER && *optr <= TSDB_RELATION_GREATER_EQUAL && *optr != TSDB_RELATION_EQUAL) { SWAP(*pLeft, *pRight, tExprNode *); } switch (*optr) { - case TSDB_RELATION_LARGE: + case TSDB_RELATION_GREATER: (*optr) = TSDB_RELATION_LESS; break; case TSDB_RELATION_LESS: - (*optr) = TSDB_RELATION_LARGE; + (*optr) = TSDB_RELATION_GREATER; break; - case TSDB_RELATION_LARGE_EQUAL: + case TSDB_RELATION_GREATER_EQUAL: (*optr) = TSDB_RELATION_LESS_EQUAL; break; case TSDB_RELATION_LESS_EQUAL: - (*optr) = TSDB_RELATION_LARGE_EQUAL; + (*optr) = TSDB_RELATION_GREATER_EQUAL; break; default:; // for other type of operations, do nothing @@ -163,9 +164,9 @@ uint8_t getBinaryExprOptr(SSQLToken *pToken) { case TK_LE: return TSDB_RELATION_LESS_EQUAL; case TK_GT: - return TSDB_RELATION_LARGE; + return TSDB_RELATION_GREATER; case TK_GE: - return TSDB_RELATION_LARGE_EQUAL; + return TSDB_RELATION_GREATER_EQUAL; case TK_NE: return TSDB_RELATION_NOT_EQUAL; case TK_AND: @@ -376,12 +377,12 @@ static char *tSQLOptrToString(uint8_t optr, char *dst) { dst += 1; break; } - case TSDB_RELATION_LARGE: { + case TSDB_RELATION_GREATER: { *dst = '>'; dst += 1; break; } - case TSDB_RELATION_LARGE_EQUAL: { + case TSDB_RELATION_GREATER_EQUAL: { *dst = '>'; *(dst + 1) = '='; dst += 2; @@ -466,8 +467,18 @@ void tExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { *pExpr = NULL; } +typedef struct { + tVariant v; + int32_t optr; +} SEndPoint; + +typedef struct { + SEndPoint* start; + SEndPoint* end; +} SQueryCond; + //static void setInitialValueForRangeQueryCondition(tSKipListQueryCond *q, int8_t type) { -// q->lowerBndRelOptr = TSDB_RELATION_LARGE; +// q->lowerBndRelOptr = TSDB_RELATION_GREATER; // q->upperBndRelOptr = TSDB_RELATION_LESS; // // switch (type) { @@ -504,60 +515,100 @@ void tExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { // } //} -//static void tSQLDoFilterInitialResult(tSkipList *pSkipList, bool (*fp)(), tQueryInfo *queryColInfo, -// tQueryResultset *result) { -// // primary key filter, search according to skiplist -// if (queryColInfo->colIdx == 0 && queryColInfo->optr != TSDB_RELATION_LIKE) { -// tSKipListQueryCond q; -// setInitialValueForRangeQueryCondition(&q, queryColInfo->q.nType); -// -// switch (queryColInfo->optr) { -// case TSDB_RELATION_EQUAL: { -// result->num = -// tSkipListPointQuery(pSkipList, &queryColInfo->q, 1, INCLUDE_POINT_QUERY, (tSkipListNode ***)&result->pRes); -// break; -// } -// case TSDB_RELATION_NOT_EQUAL: { -// result->num = -// tSkipListPointQuery(pSkipList, &queryColInfo->q, 1, EXCLUDE_POINT_QUERY, (tSkipListNode ***)&result->pRes); -// break; -// } -// case TSDB_RELATION_LESS_EQUAL: { -// tVariantAssign(&q.upperBnd, &queryColInfo->q); -// q.upperBndRelOptr = queryColInfo->optr; -// result->num = tSkipListQuery(pSkipList, &q, (tSkipListNode ***)&result->pRes); -// break; -// } -// case TSDB_RELATION_LESS: { -// tVariantAssign(&q.upperBnd, &queryColInfo->q); -// result->num = tSkipListQuery(pSkipList, &q, (tSkipListNode ***)&result->pRes); -// break; -// } -// case TSDB_RELATION_LARGE: { -// tVariantAssign(&q.lowerBnd, &queryColInfo->q); -// result->num = tSkipListQuery(pSkipList, &q, (tSkipListNode ***)&result->pRes); -// break; -// } -// case TSDB_RELATION_LARGE_EQUAL: { -// tVariantAssign(&q.lowerBnd, &queryColInfo->q); -// q.lowerBndRelOptr = queryColInfo->optr; -// result->num = tSkipListQuery(pSkipList, &q, (tSkipListNode ***)&result->pRes); -// break; -// } -// default: -// pTrace("skiplist:%p, unsupport query operator:%d", pSkipList, queryColInfo->optr); -// } -// -// tSkipListDestroyKey(&q.upperBnd); -// tSkipListDestroyKey(&q.lowerBnd); -// } else { -// /* -// * Brutal force scan the whole skiplit to find the appropriate result, -// * since the filter is not applied to indexed column. -// */ -// result->num = tSkipListIterateList(pSkipList, (tSkipListNode ***)&result->pRes, fp, queryColInfo); -// } -//} +// todo check for malloc failure +static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) { + int32_t optr = queryColInfo->optr; + + if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL || + optr == TSDB_RELATION_EQUAL || optr == TSDB_RELATION_NOT_EQUAL) { + pCond->start = calloc(1, sizeof(tVariant)); + + tVariantAssign(&pCond->start->v, &queryColInfo->q); + pCond->start->optr = queryColInfo->optr; + } else if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { + pCond->end = calloc(1, sizeof(tVariant)); + + tVariantAssign(&pCond->end->v, &queryColInfo->q); + pCond->end->optr = queryColInfo->optr; + } + + return TSDB_CODE_SUCCESS; +} + +static void tQueryOnSkipList(SSkipList* pSkipList, SQueryCond* pCond, int32_t type, SArray* result) { + SSkipListIterator* iter = NULL; + + if (pCond->start != NULL) { + iter = tSkipListCreateIterFromVal(pSkipList, (char*) &pCond->start->v.i64Key, type, TSDB_ORDER_ASC); + } else { + iter = tSkipListCreateIterFromVal(pSkipList, (char*) &pCond->end->v.i64Key, type, TSDB_ORDER_DESC); + } + + __compar_fn_t func = getComparFunc(pSkipList->keyInfo.type, type); + + if (pCond->start != NULL) { + int32_t optr = pCond->start->optr; + + if (optr == TSDB_RELATION_EQUAL) { + while(tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + + int32_t ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &pCond->start->v.i64Key); + if (ret == 0) { + taosArrayPush(result, SL_GET_NODE_DATA(pNode)); + } else { + break; + } + } + } else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) { + bool comp = true; + int32_t ret = 0; + + while(tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + + if (comp) { + ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &pCond->start->v.i64Key); + assert(ret <= 0); + } + + if (ret == 0 && optr == TSDB_RELATION_GREATER) { + continue; + } else { + taosArrayPush(result, SL_GET_NODE_DATA(pNode)); + comp = false; + } + } + } else if (optr == TSDB_RELATION_NOT_EQUAL) { + assert(0); + } else { + assert(0); + } + } else { + int32_t optr = pCond->end->optr; + + if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { + bool comp = true; + int32_t ret = 0; + + while(tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + + if (comp) { + ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &pCond->start->v.i64Key); + assert(ret >= 0); + } + + if (ret == 0 && optr == TSDB_RELATION_LESS) { + continue; + } else { + taosArrayPush(result, SL_GET_NODE_DATA(pNode)); + comp = false; // no need to compare anymore + } + } + } + } +} /* * qsort comparator @@ -670,9 +721,7 @@ static bool filterItem(tExprNode *pExpr, const void *pItem, SBinaryFilterSupp *p tExprNode *pLeft = pExpr->_node.pLeft; tExprNode *pRight = pExpr->_node.pRight; - /* - * non-leaf nodes, recursively traverse the syntax tree in the post-root order - */ + //non-leaf nodes, recursively traverse the expression tree in the post-root order if (pLeft->nodeType == TSQL_NODE_EXPR && pRight->nodeType == TSQL_NODE_EXPR) { if (pExpr->_node.optr == TSDB_RELATION_OR) { // or if (filterItem(pLeft, pItem, param)) { @@ -707,7 +756,7 @@ static bool filterItem(tExprNode *pExpr, const void *pItem, SBinaryFilterSupp *p * @param pSchema tag schemas * @param fp filter callback function */ -static void tSQLBinaryTraverseOnResult(tExprNode *pExpr, SArray *pResult, SBinaryFilterSupp *param) { +static void exprTreeTraverseImpl(tExprNode *pExpr, SArray *pResult, SBinaryFilterSupp *param) { size_t size = taosArrayGetSize(pResult); SArray* array = taosArrayInit(size, POINTER_BYTES); @@ -736,12 +785,12 @@ static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSki } // post-root order traverse syntax tree -void tSQLBinaryExprTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param) { +void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param) { if (pExpr == NULL) { return; } - tExprNode *pLeft = pExpr->_node.pLeft; + tExprNode *pLeft = pExpr->_node.pLeft; tExprNode *pRight = pExpr->_node.pRight; // recursive traverse left child branch @@ -755,7 +804,7 @@ void tSQLBinaryExprTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *resu * * if the query is a high selectivity filter, only small portion of meters are retrieved. */ - tSQLBinaryTraverseOnResult(pExpr, result, param); + exprTreeTraverseImpl(pExpr, result, param); } else if (weight == 0) { /** * apply the hierarchical expression to every node in skiplist for find the qualified nodes @@ -766,8 +815,8 @@ void tSQLBinaryExprTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *resu SArray* rLeft = taosArrayInit(10, POINTER_BYTES); SArray* rRight = taosArrayInit(10, POINTER_BYTES); - tSQLBinaryExprTraverse(pLeft, pSkipList, rLeft, param); - tSQLBinaryExprTraverse(pRight, pSkipList, rRight, param); + tExprTreeTraverse(pLeft, pSkipList, rLeft, param); + tExprTreeTraverse(pRight, pSkipList, rRight, param); if (pExpr->_node.optr == TSDB_RELATION_AND) { // CROSS intersect(rLeft, rRight, result); @@ -801,13 +850,13 @@ void tSQLBinaryExprTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *resu assert(pFirst != pSecond && pFirst != NULL && pSecond != NULL); // we filter the result based on the skiplist index in the first place - tSQLBinaryExprTraverse(pFirst, pSkipList, result, param); + tExprTreeTraverse(pFirst, pSkipList, result, param); /* * recursively perform the filter operation based on the initial results, - * So, we do not set the skiplist index as a parameter + * So, we do not set the skip list index as a parameter */ - tSQLBinaryExprTraverse(pSecond, NULL, result, param); + tExprTreeTraverse(pSecond, NULL, result, param); } } else { // column project assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE); @@ -816,8 +865,19 @@ void tSQLBinaryExprTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *resu if (pSkipList == NULL) { tSQLListTraverseOnResult(pExpr, param->fp, result); } else { -// assert(result->num == 0); -// tSQLDoFilterInitialResult(pSkipList, param->fp, pExpr->info, result); + tQueryInfo *pQueryInfo = pExpr->_node.info; + + if (pQueryInfo->colIndex == 0 && pQueryInfo->optr != TSDB_RELATION_LIKE) { + SQueryCond cond = {0}; + /*int32_t ret = */setQueryCond(pQueryInfo, &cond); + tQueryOnSkipList(pSkipList, &cond, pQueryInfo->q.nType, result); + } else { + /* Brutal force scan the whole skip list to find the appropriate result, + * since the filter is not applied to indexed column. + */ + assert(0); +// result->num = tSkipListIterateList(pSkipList, (tSkipListNode ***)&result->pRes, fp, queryColInfo); + } } } } @@ -1012,11 +1072,10 @@ static void exprTreeFromBinaryImpl(tExprNode** pExprTree, SBuffer* pBuf) { *pExprTree = pExpr; } -tExprNode* exprTreeFromBinary(const void* pBuf, size_t size) { +int32_t exprTreeFromBinary(const void* pBuf, size_t size, tExprNode** pExprNode) { SBuffer rbuf = {0}; /*int32_t code =*/ tbufBeginRead(&rbuf, pBuf, size); - - tExprNode* pExprNode = NULL; - exprTreeFromBinaryImpl(&pExprNode, &rbuf); - return pExprNode; -} \ No newline at end of file + exprTreeFromBinaryImpl(pExprNode, &rbuf); + return TSDB_CODE_SUCCESS; +} + diff --git a/src/query/src/qextbuffer.c b/src/query/src/qextbuffer.c index d71d5669996ef306b7185d899c3eab52f893aa61..0195ae8b267464cebe346a3485fcf2185df115b6 100644 --- a/src/query/src/qextbuffer.c +++ b/src/query/src/qextbuffer.c @@ -345,7 +345,7 @@ static FORCE_INLINE int32_t primaryKeyComparator(int64_t f1, int64_t f2, int32_t return 0; } - if (colIdx == 0 && tsOrder == TSQL_SO_DESC) { // primary column desc order + if (colIdx == 0 && tsOrder == TSDB_ORDER_DESC) { // primary column desc order return (f1 < f2) ? 1 : -1; } else { // asc return (f1 < f2) ? -1 : 1; @@ -628,7 +628,7 @@ static int32_t qsort_call = 0; void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data, int32_t orderType) { // short array sort, incur another sort procedure instead of quick sort process - __col_compar_fn_t compareFn = (orderType == TSQL_SO_ASC) ? compare_sa : compare_sd; + __col_compar_fn_t compareFn = (orderType == TSDB_ORDER_ASC) ? compare_sa : compare_sd; if (end - start + 1 <= 8) { tColDataInsertSort(pDescriptor, numOfRows, start, end, data, compareFn); diff --git a/src/query/src/qinterpolation.c b/src/query/src/qinterpolation.c index 1731e16ed82703753c59d2a8f34411b39eba8c3b..40fcf63c363912ff9f5eefaaf98ecdf91eaebf26 100644 --- a/src/query/src/qinterpolation.c +++ b/src/query/src/qinterpolation.c @@ -20,7 +20,7 @@ #include "taosmsg.h" #include "tsqlfunction.h" -#define INTERPOL_IS_ASC_INTERPOL(interp) ((interp)->order == TSQL_SO_ASC) +#define INTERPOL_IS_ASC_INTERPOL(interp) ((interp)->order == TSDB_ORDER_ASC) int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t timeRange, char intervalTimeUnit, int16_t precision) { if (timeRange == 0) { @@ -96,7 +96,7 @@ void taosInterpoSetStartInfo(SInterpolationInfo* pInterpoInfo, int32_t numOfRawD } TSKEY taosGetRevisedEndKey(TSKEY ekey, int32_t order, int32_t timeInterval, int8_t intervalTimeUnit, int8_t precision) { - if (order == TSQL_SO_ASC) { + if (order == TSDB_ORDER_ASC) { return ekey; } else { return taosGetIntervalStartTimestamp(ekey, timeInterval, intervalTimeUnit, precision); diff --git a/src/query/src/qpercentile.c b/src/query/src/qpercentile.c index 3b12dee0538200903ae19f2a364ea2b75543a1ae..7283a9fd3ffa71d8eb2fff265da3b989259084eb 100644 --- a/src/query/src/qpercentile.c +++ b/src/query/src/qpercentile.c @@ -83,7 +83,7 @@ static tFilePage *loadIntoBucketFromDisk(tMemBucket *pMemBucket, int32_t segIdx, pListItem = pListItem->pNext; } - tColDataQSort(pDesc, buffer->numOfElems, 0, buffer->numOfElems - 1, buffer->data, TSQL_SO_ASC); + tColDataQSort(pDesc, buffer->numOfElems, 0, buffer->numOfElems - 1, buffer->data, TSDB_ORDER_ASC); pDesc->pColumnModel->capacity = oldCapacity; // restore value return buffer; diff --git a/src/query/src/qsyntaxtreefunction.c b/src/query/src/qsyntaxtreefunction.c index d21f7dab736b1b14579f3b207da89695e93a1077..41e84b5ab02bab0599ae95388780364edf32d672 100644 --- a/src/query/src/qsyntaxtreefunction.c +++ b/src/query/src/qsyntaxtreefunction.c @@ -21,8 +21,8 @@ #define ARRAY_LIST_OP(left, right, _left_type, _right_type, len1, len2, out, op, _res_type, _ord) \ { \ - int32_t i = ((_ord) == TSQL_SO_ASC) ? 0 : MAX(len1, len2) - 1; \ - int32_t step = ((_ord) == TSQL_SO_ASC) ? 1 : -1; \ + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(len1, len2) - 1; \ + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; \ \ if ((len1) == (len2)) { \ for (; i < (len2) && i >= 0; i += step, (out) += 1) { \ @@ -53,8 +53,8 @@ #define ARRAY_LIST_OP_REM(left, right, _left_type, _right_type, len1, len2, out, op, _res_type, _ord) \ { \ - int32_t i = (_ord == TSQL_SO_ASC) ? 0 : MAX(len1, len2) - 1; \ - int32_t step = (_ord == TSQL_SO_ASC) ? 1 : -1; \ + int32_t i = (_ord == TSDB_ORDER_ASC) ? 0 : MAX(len1, len2) - 1; \ + int32_t step = (_ord == TSDB_ORDER_ASC) ? 1 : -1; \ \ if (len1 == (len2)) { \ for (; i >= 0 && i < (len2); i += step, (out) += 1) { \ @@ -107,8 +107,8 @@ void calc_fn_i32_i32_add(void *left, void *right, int32_t numLeft, int32_t numRi int32_t *pRight = (int32_t *)right; double * pOutput = (double *)output; - int32_t i = (order == TSQL_SO_ASC) ? 0 : MAX(numLeft, numRight) - 1; - int32_t step = (order == TSQL_SO_ASC) ? 1 : -1; + int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; + int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; if (numLeft == numRight) { for (; i >= 0 && i < numRight; i += step, pOutput += 1) { @@ -305,8 +305,8 @@ void calc_fn_i32_i32_sub(void *left, void *right, int32_t numLeft, int32_t numRi int32_t *pRight = (int32_t *)right; double * pOutput = (double *)output; - int32_t i = (order == TSQL_SO_ASC) ? 0 : MAX(numLeft, numRight) - 1; - int32_t step = (order == TSQL_SO_ASC) ? 1 : -1; + int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; + int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; if (numLeft == numRight) { for (; i >= 0 && i < numRight; i += step, pOutput += 1) { @@ -516,8 +516,8 @@ void calc_fn_i32_i32_multi(void *left, void *right, int32_t numLeft, int32_t num int32_t *pRight = (int32_t *)right; double * pOutput = (double *)output; - int32_t i = (order == TSQL_SO_ASC) ? 0 : MAX(numLeft, numRight) - 1; - int32_t step = (order == TSQL_SO_ASC) ? 1 : -1; + int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; + int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; if (numLeft == numRight) { for (; i >= 0 && i < numRight; i += step, pOutput += 1) { @@ -714,8 +714,8 @@ void calc_fn_i32_i32_div(void *left, void *right, int32_t numLeft, int32_t numRi int32_t *pRight = (int32_t *)right; double * pOutput = (double *)output; - int32_t i = (order == TSQL_SO_ASC) ? 0 : MAX(numLeft, numRight) - 1; - int32_t step = (order == TSQL_SO_ASC) ? 1 : -1; + int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; + int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; if (numLeft == numRight) { for (; i >= 0 && i < numRight; i += step, pOutput += 1) { @@ -928,8 +928,8 @@ void calc_fn_i32_i32_rem(void *left, void *right, int32_t numLeft, int32_t numRi int32_t *pRight = (int32_t *)right; double * pOutput = (double *)output; - int32_t i = (order == TSQL_SO_ASC) ? 0 : MAX(numLeft, numRight) - 1; - int32_t step = (order == TSQL_SO_ASC) ? 1 : -1; + int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; + int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; if (numLeft == numRight) { for (; i >= 0 && i < numRight; i += step, pOutput += 1) { @@ -986,8 +986,8 @@ void calc_fn_i32_d_rem(void *left, void *right, int32_t numLeft, int32_t numRigh double * pRight = (double *)right; double * pOutput = (double *)output; - int32_t i = (order == TSQL_SO_ASC) ? 0 : MAX(numLeft, numRight) - 1; - int32_t step = (order == TSQL_SO_ASC) ? 1 : -1; + int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; + int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; if (numLeft == numRight) { for (; i >= 0 && i < numRight; i += step, pOutput += 1) { diff --git a/src/query/src/qtsbuf.c b/src/query/src/qtsbuf.c index ea6e6dfdc0f6db5f18b2254887df028f75233fa6..062a8038b2699a7be77e7eca3290fedfe9321785 100644 --- a/src/query/src/qtsbuf.c +++ b/src/query/src/qtsbuf.c @@ -33,11 +33,11 @@ STSBuf* tsBufCreate(bool autoDelete) { } // update the header info - STSBufFileHeader header = {.magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = TSQL_SO_ASC}; + STSBufFileHeader header = {.magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = TSDB_ORDER_ASC}; STSBufUpdateHeader(pTSBuf, &header); tsBufResetPos(pTSBuf); - pTSBuf->cur.order = TSQL_SO_ASC; + pTSBuf->cur.order = TSDB_ORDER_ASC; pTSBuf->autoDelete = autoDelete; pTSBuf->tsOrder = -1; @@ -88,7 +88,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { // check the ts order pTSBuf->tsOrder = header.tsOrder; - if (pTSBuf->tsOrder != TSQL_SO_ASC && pTSBuf->tsOrder != TSQL_SO_DESC) { + if (pTSBuf->tsOrder != TSDB_ORDER_ASC && pTSBuf->tsOrder != TSDB_ORDER_DESC) { // tscError("invalid order info in buf:%d", pTSBuf->tsOrder); tsBufDestory(pTSBuf); return NULL; @@ -118,7 +118,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { tsBufResetPos(pTSBuf); // ascending by default - pTSBuf->cur.order = TSQL_SO_ASC; + pTSBuf->cur.order = TSDB_ORDER_ASC; pTSBuf->autoDelete = autoDelete; @@ -274,7 +274,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { memset(pBlock, 0, sizeof(STSBlock)); pBlock->payload = tmp; - if (order == TSQL_SO_DESC) { + if (order == TSDB_ORDER_DESC) { /* * set the right position for the reversed traverse, the reversed traverse is started from * the end of each comp data block @@ -303,7 +303,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f); // for backwards traverse, set the start position at the end of previous block - if (order == TSQL_SO_DESC) { + if (order == TSDB_ORDER_DESC) { int32_t offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag); int64_t r = fseek(pTSBuf->f, -offset, SEEK_CUR); UNUSED(r); @@ -321,9 +321,9 @@ static int32_t setCheckTSOrder(STSBuf* pTSBuf, const char* pData, int32_t len) { TSKEY lastKey = *(TSKEY*)(ptsData->rawBuf + ptsData->len - TSDB_KEYSIZE); if (lastKey > *(TSKEY*)pData) { - pTSBuf->tsOrder = TSQL_SO_DESC; + pTSBuf->tsOrder = TSDB_ORDER_DESC; } else { - pTSBuf->tsOrder = TSQL_SO_ASC; + pTSBuf->tsOrder = TSDB_ORDER_ASC; } } else if (len > TSDB_KEYSIZE) { // no data in current vnode, more than one ts is added, check the orders @@ -331,9 +331,9 @@ static int32_t setCheckTSOrder(STSBuf* pTSBuf, const char* pData, int32_t len) { TSKEY k2 = *(TSKEY*)(pData + TSDB_KEYSIZE); if (k1 < k2) { - pTSBuf->tsOrder = TSQL_SO_ASC; + pTSBuf->tsOrder = TSDB_ORDER_ASC; } else if (k1 > k2) { - pTSBuf->tsOrder = TSQL_SO_DESC; + pTSBuf->tsOrder = TSDB_ORDER_DESC; } else { // todo handle error } @@ -432,13 +432,13 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int bool decomp = false; while ((i++) <= blockIndex) { - if (readDataFromDisk(pTSBuf, TSQL_SO_ASC, decomp) == NULL) { + if (readDataFromDisk(pTSBuf, TSDB_ORDER_ASC, decomp) == NULL) { return -1; } } // set the file position to be the end of previous comp block - if (pTSBuf->cur.order == TSQL_SO_DESC) { + if (pTSBuf->cur.order == TSDB_ORDER_DESC) { STSBlock* pBlock = &pTSBuf->block; int32_t compBlockSize = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag); @@ -452,7 +452,7 @@ static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo bool decomp = false; int64_t offset = 0; - if (pTSBuf->cur.order == TSQL_SO_ASC) { + if (pTSBuf->cur.order == TSDB_ORDER_ASC) { offset = pBlockInfo->offset; } else { // reversed traverse starts from the end of block offset = pBlockInfo->offset + pBlockInfo->compLen; @@ -482,8 +482,8 @@ static void tsBufGetBlock(STSBuf* pTSBuf, int32_t vnodeIndex, int32_t blockIndex } STSCursor* pCur = &pTSBuf->cur; - if (pCur->vnodeIndex == vnodeIndex && ((pCur->blockIndex <= blockIndex && pCur->order == TSQL_SO_ASC) || - (pCur->blockIndex >= blockIndex && pCur->order == TSQL_SO_DESC))) { + if (pCur->vnodeIndex == vnodeIndex && ((pCur->blockIndex <= blockIndex && pCur->order == TSDB_ORDER_ASC) || + (pCur->blockIndex >= blockIndex && pCur->order == TSDB_ORDER_DESC))) { int32_t i = 0; bool decomp = false; int32_t step = abs(blockIndex - pCur->blockIndex); @@ -520,7 +520,7 @@ static void tsBufGetBlock(STSBuf* pTSBuf, int32_t vnodeIndex, int32_t blockIndex pCur->vnodeIndex = vnodeIndex; pCur->blockIndex = blockIndex; - pCur->tsIndex = (pCur->order == TSQL_SO_ASC) ? 0 : pBlock->numOfElem - 1; + pCur->tsIndex = (pCur->order == TSDB_ORDER_ASC) ? 0 : pBlock->numOfElem - 1; } STSVnodeBlockInfo* tsBufGetVnodeBlockInfo(STSBuf* pTSBuf, int32_t vnodeId) { @@ -555,7 +555,7 @@ bool tsBufNextPos(STSBuf* pTSBuf) { // get the first/last position according to traverse order if (pCur->vnodeIndex == -1) { - if (pCur->order == TSQL_SO_ASC) { + if (pCur->order == TSDB_ORDER_ASC) { tsBufGetBlock(pTSBuf, 0, 0); if (pTSBuf->block.numOfElem == 0) { // the whole list is empty, return @@ -587,20 +587,20 @@ bool tsBufNextPos(STSBuf* pTSBuf) { } } - int32_t step = pCur->order == TSQL_SO_ASC ? 1 : -1; + int32_t step = pCur->order == TSDB_ORDER_ASC ? 1 : -1; while (1) { assert(pTSBuf->tsData.len == pTSBuf->block.numOfElem * TSDB_KEYSIZE); - if ((pCur->order == TSQL_SO_ASC && pCur->tsIndex >= pTSBuf->block.numOfElem - 1) || - (pCur->order == TSQL_SO_DESC && pCur->tsIndex <= 0)) { + if ((pCur->order == TSDB_ORDER_ASC && pCur->tsIndex >= pTSBuf->block.numOfElem - 1) || + (pCur->order == TSDB_ORDER_DESC && pCur->tsIndex <= 0)) { int32_t vnodeId = pTSBuf->pData[pCur->vnodeIndex].info.vnode; STSVnodeBlockInfo* pBlockInfo = tsBufGetVnodeBlockInfo(pTSBuf, vnodeId); - if (pBlockInfo == NULL || (pCur->blockIndex >= pBlockInfo->numOfBlocks - 1 && pCur->order == TSQL_SO_ASC) || - (pCur->blockIndex <= 0 && pCur->order == TSQL_SO_DESC)) { - if ((pCur->vnodeIndex >= pTSBuf->numOfVnodes - 1 && pCur->order == TSQL_SO_ASC) || - (pCur->vnodeIndex <= 0 && pCur->order == TSQL_SO_DESC)) { + if (pBlockInfo == NULL || (pCur->blockIndex >= pBlockInfo->numOfBlocks - 1 && pCur->order == TSDB_ORDER_ASC) || + (pCur->blockIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) { + if ((pCur->vnodeIndex >= pTSBuf->numOfVnodes - 1 && pCur->order == TSDB_ORDER_ASC) || + (pCur->vnodeIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) { pCur->vnodeIndex = -1; return false; } @@ -609,7 +609,7 @@ bool tsBufNextPos(STSBuf* pTSBuf) { return false; } - int32_t blockIndex = pCur->order == TSQL_SO_ASC ? 0 : pBlockInfo->numOfBlocks - 1; + int32_t blockIndex = pCur->order == TSDB_ORDER_ASC ? 0 : pBlockInfo->numOfBlocks - 1; tsBufGetBlock(pTSBuf, pCur->vnodeIndex + step, blockIndex); break; @@ -766,7 +766,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ pTSBuf->fileSize += len; pTSBuf->tsOrder = order; - assert(order == TSQL_SO_ASC || order == TSQL_SO_DESC); + assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); STSBufFileHeader header = { .magic = TS_COMP_FILE_MAGIC, .numOfVnode = pTSBuf->numOfVnodes, .tsOrder = pTSBuf->tsOrder}; @@ -850,7 +850,7 @@ void tsBufDisplay(STSBuf* pTSBuf) { printf("number of vnode:%d\n", pTSBuf->numOfVnodes); int32_t old = pTSBuf->cur.order; - pTSBuf->cur.order = TSQL_SO_ASC; + pTSBuf->cur.order = TSDB_ORDER_ASC; tsBufResetPos(pTSBuf); diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 5af0cdc687866b051612caf5ff96c96743c7daaa..9cb2dcd6c69a7f85570808b9dfdce48002ce069d 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -52,9 +52,9 @@ /* get the qinfo struct address from the query struct address */ #define GET_COLUMN_BYTES(query, colidx) \ - ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIdx].info.bytes) + ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIndex].info.bytes) #define GET_COLUMN_TYPE(query, colidx) \ - ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIdx].info.type) + ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIndex].info.type) typedef struct SPointInterpoSupporter { int32_t numOfCols; @@ -279,6 +279,7 @@ int64_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv) { } } + assert(maxOutput >= 0); return maxOutput; } @@ -293,13 +294,13 @@ bool isGroupbyNormalCol(SSqlGroupbyExpr *pGroupbyExpr) { } for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { - SColIndexEx *pColIndex = &pGroupbyExpr->columnInfo[i]; + SColIndex *pColIndex = &pGroupbyExpr->columnInfo[i]; if (pColIndex->flag == TSDB_COL_NORMAL) { /* * make sure the normal column locates at the second position if tbname exists in group by clause */ if (pGroupbyExpr->numOfGroupCols > 1) { - assert(pColIndex->colIdx > 0); + assert(pColIndex->colIndex > 0); } return true; @@ -316,7 +317,7 @@ int16_t getGroupbyColumnType(SQuery *pQuery, SSqlGroupbyExpr *pGroupbyExpr) { int16_t type = TSDB_DATA_TYPE_NULL; for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { - SColIndexEx *pColIndex = &pGroupbyExpr->columnInfo[i]; + SColIndex *pColIndex = &pGroupbyExpr->columnInfo[i]; if (pColIndex->flag == TSDB_COL_NORMAL) { colId = pColIndex->colId; break; @@ -361,8 +362,8 @@ bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].pBase.functio bool doRevisedResultsByLimit(SQInfo *pQInfo) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - if ((pQuery->limit.limit > 0) && (pQuery->rec.size + pQuery->rec.size > pQuery->limit.limit)) { - pQuery->rec.size = pQuery->limit.limit - pQuery->rec.size; + if ((pQuery->limit.limit > 0) && (pQuery->rec.rows + pQuery->rec.rows > pQuery->limit.limit)) { + pQuery->rec.rows = pQuery->limit.limit - pQuery->rec.rows; // query completed setQueryStatus(pQuery, QUERY_COMPLETED); @@ -389,7 +390,7 @@ static bool isTopBottomQuery(SQuery *pQuery) { static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, int32_t index) { // for a tag column, no corresponding field info - SColIndexEx *pColIndexEx = &pQuery->pSelectExpr[index].pBase.colInfo; + SColIndex *pColIndexEx = &pQuery->pSelectExpr[index].pBase.colInfo; if (TSDB_COL_IS_TAG(pColIndexEx->flag)) { return NULL; } @@ -417,7 +418,7 @@ static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlo */ static bool hasNullValue(SQuery *pQuery, int32_t col, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, SDataStatis **pColStatis) { - SColIndexEx* pColIndex = &pQuery->pSelectExpr[col].pBase.colInfo; + SColIndex* pColIndex = &pQuery->pSelectExpr[col].pBase.colInfo; if (TSDB_COL_IS_TAG(pColIndex->flag)) { return false; } @@ -592,7 +593,7 @@ static int32_t getForwardStepsInBlock(int32_t numOfPoints, __block_search_fn_t s int32_t forwardStep = 0; if (endPos >= 0) { - forwardStep = (order == TSQL_SO_ASC) ? (endPos - pos) : (pos - endPos); + forwardStep = (order == TSDB_ORDER_ASC) ? (endPos - pos) : (pos - endPos); assert(forwardStep >= 0); // endPos data is equalled to the key so, we do need to read the element in endPos @@ -669,7 +670,7 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn, int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, bool updateLastKey) { - assert(startPos >= 0 && startPos < pDataBlockInfo->size); + assert(startPos >= 0 && startPos < pDataBlockInfo->rows); int32_t num = -1; int32_t order = pQuery->order.order; @@ -678,7 +679,7 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo if (QUERY_IS_ASC_QUERY(pQuery)) { if (ekey < pDataBlockInfo->window.ekey) { - num = getForwardStepsInBlock(pDataBlockInfo->size, searchFn, ekey, startPos, order, pPrimaryColumn); + num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); if (num == 0) { // no qualified data in current block, do not update the lastKey value assert(ekey < pPrimaryColumn[startPos]); } else { @@ -687,14 +688,14 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo } } } else { - num = pDataBlockInfo->size - startPos; + num = pDataBlockInfo->rows - startPos; if (updateLastKey) { pQuery->lastKey = pDataBlockInfo->window.ekey + step; } } } else { // desc if (ekey > pDataBlockInfo->window.skey) { - num = getForwardStepsInBlock(pDataBlockInfo->size, searchFn, ekey, startPos, order, pPrimaryColumn); + num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); if (num == 0) { // no qualified data in current block, do not update the lastKey value assert(ekey > pPrimaryColumn[startPos]); } else { @@ -787,7 +788,7 @@ static int32_t getNextQualifiedWindow(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow } } - int32_t startPos = searchFn((char *)primaryKeys, pDataBlockInfo->size, startKey, pQuery->order.order); + int32_t startPos = searchFn((char *)primaryKeys, pDataBlockInfo->rows, startKey, pQuery->order.order); /* * This time window does not cover any data, try next time window, @@ -850,12 +851,12 @@ char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int3 sas->numOfCols = pQuery->numOfCols; sas->offset = 0; } else { // other type of query function - SColIndexEx *pCol = &pQuery->pSelectExpr[col].pBase.colInfo; + SColIndex *pCol = &pQuery->pSelectExpr[col].pBase.colInfo; if (TSDB_COL_IS_TAG(pCol->flag)) { dataBlock = NULL; } else { /* - * the colIdx is acquired from the first meter of all qualified meters in this vnode during query prepare stage, + * the colIndex is acquired from the first meter of all qualified meters in this vnode during query prepare stage, * the remain meter may not have the required column in cache actually. * So, the validation of required column in cache with the corresponding meter schema is reinforced. */ @@ -887,7 +888,7 @@ char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int3 * @return the incremental number of output value, so it maybe 0 for fixed number of query, * such as count/min/max etc. */ -static int32_t blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, +static void blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, SWindowResInfo *pWindowResInfo, __block_search_fn_t searchFn, SArray *pDataBlock) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; @@ -901,9 +902,7 @@ static int32_t blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataSt primaryKeyCol = (TSKEY *)(pColInfo->pData); } - pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : pDataBlockInfo->size - 1; - int64_t prevNumOfRes = getNumOfResult(pRuntimeEnv); - + pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : pDataBlockInfo->rows - 1; SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutputCols, sizeof(SArithmeticSupport)); for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { @@ -912,9 +911,9 @@ static int32_t blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataSt SDataStatis *tpField = NULL; bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &tpField); - char *dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->size, pDataBlock); + char *dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->size, functionId, tpField, + setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, tpField, hasNull, &sasArray[k], pRuntimeEnv->scanFlag); } @@ -924,11 +923,9 @@ static int32_t blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataSt TSKEY ts = primaryKeyCol[offset]; STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); - assert(0); - // if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pRuntimeEnv->pTabObj->sid, &win) != - // TSDB_CODE_SUCCESS) { - // return 0; - // } + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->sid, &win) != TSDB_CODE_SUCCESS) { + return; + } TSKEY ekey = reviseWindowEkey(pQuery, &win); int32_t forwardStep = @@ -948,10 +945,7 @@ static int32_t blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataSt } // null data, failed to allocate more memory buffer - // int32_t sid = pRuntimeEnv->pTabObj->sid; - int32_t sid = 0; - assert(0); - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, sid, &nextWin) != TSDB_CODE_SUCCESS) { + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->sid, &nextWin) != TSDB_CODE_SUCCESS) { break; } @@ -959,7 +953,6 @@ static int32_t blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataSt forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, primaryKeyCol, startPos, ekey, searchFn, true); pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); - doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, startPos, forwardStep, primaryKeyCol); } @@ -977,18 +970,8 @@ static int32_t blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataSt } } } - - /* - * No need to calculate the number of output results for group-by normal columns, interval query - * because the results of group by normal column is put into intermediate buffer. - */ - int32_t num = 0; - if (!isIntervalQuery(pQuery)) { - num = getNumOfResult(pRuntimeEnv) - prevNumOfRes; - } - + tfree(sasArray); - return (int32_t)num; } static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pData, int16_t type, int16_t bytes) { @@ -1137,9 +1120,9 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat SDataStatis *pColStatis = NULL; bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &pColStatis); - char *dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->size, pDataBlock); + char *dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->size, functionId, pColStatis, + setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, pColStatis, hasNull, &sasArray[k], pRuntimeEnv->scanFlag); } @@ -1161,14 +1144,14 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat // from bottom to top in asc order if (pRuntimeEnv->pTSBuf != NULL) { SQInfo *pQInfo = (SQInfo *)GET_QINFO_ADDR(pQuery); - qTrace("QInfo:%p process data rows, numOfRows:%d, query order:%d, ts comp order:%d", pQInfo, pDataBlockInfo->size, + qTrace("QInfo:%p process data rows, numOfRows:%d, query order:%d, ts comp order:%d", pQInfo, pDataBlockInfo->rows, pQuery->order.order, pRuntimeEnv->pTSBuf->cur.order); } int32_t j = 0; TSKEY lastKey = -1; - for (j = 0; j < pDataBlockInfo->size; ++j) { + for (j = 0; j < pDataBlockInfo->rows; ++j) { int32_t offset = GET_COL_DATA_POS(pQuery, j, step); if (pRuntimeEnv->pTSBuf != NULL) { @@ -1270,7 +1253,7 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat * pointsOffset is the maximum available space in result buffer update the actual forward step for query that * requires checking buffer during loop */ - if ((pQuery->checkBufferInLoop == 1) && (++numOfRes) >= pQuery->pointsOffset) { + if ((pQuery->checkBuffer == 1) && (++numOfRes) >= pQuery->pointsOffset) { pQuery->lastKey = lastKey + step; assert(0); // *forwardStep = j + 1; @@ -1296,7 +1279,7 @@ static UNUSED_FUNC int32_t reviseForwardSteps(SQueryRuntimeEnv *pRuntimeEnv, int /* * 1. If value filter exists, we try all data in current block, and do not set the QUERY_RESBUF_FULL flag. * - * 2. In case of top/bottom/ts_comp query, the checkBufferInLoop == 1 and pQuery->numOfFilterCols + * 2. In case of top/bottom/ts_comp query, the checkBuffer == 1 and pQuery->numOfFilterCols * may be 0 or not. We do not check the capacity of output buffer, since the filter function will do it. * * 3. In handling the query of secondary query of join, tsBuf servers as a ts filter. @@ -1308,7 +1291,7 @@ static UNUSED_FUNC int32_t reviseForwardSteps(SQueryRuntimeEnv *pRuntimeEnv, int } // current buffer does not have enough space, try in the next loop - if ((pQuery->checkBufferInLoop == 1) && (pQuery->pointsOffset <= forwardStep)) { + if ((pQuery->checkBuffer == 1) && (pQuery->pointsOffset <= forwardStep)) { forwardStep = pQuery->pointsOffset; } @@ -1316,17 +1299,17 @@ static UNUSED_FUNC int32_t reviseForwardSteps(SQueryRuntimeEnv *pRuntimeEnv, int } static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, - SDataStatis *pStatis, __block_search_fn_t searchFn, int32_t *numOfRes, - SWindowResInfo *pWindowResInfo, SArray *pDataBlock) { + SDataStatis *pStatis, __block_search_fn_t searchFn, SWindowResInfo *pWindowResInfo, SArray *pDataBlock) { SQuery *pQuery = pRuntimeEnv->pQuery; + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { - *numOfRes = rowwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); + /*numOfRes = */rowwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); } else { - *numOfRes = blockwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); + blockwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); } - TSKEY lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pDataBlockInfo->window.ekey : pDataBlockInfo->window.skey; + TSKEY lastKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey : pDataBlockInfo->window.skey; pQuery->lastKey = lastKey + GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); doCheckQueryCompleted(pRuntimeEnv, lastKey, pWindowResInfo); @@ -1338,14 +1321,19 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl setQueryStatus(pQuery, QUERY_COMPLETED); } - assert(*numOfRes >= 0); + int32_t numOfRes = getNumOfResult(pRuntimeEnv); - // check if buffer is large enough for accommodating all qualified points - if (*numOfRes > 0 && pQuery->checkBufferInLoop == 1 && ((*numOfRes) >= pQuery->rec.threshold)) { - setQueryStatus(pQuery, QUERY_RESBUF_FULL); + // update the number of output result + if (numOfRes > 0 && pQuery->checkBuffer == 1) { + assert(numOfRes >= pQuery->rec.rows); + pQuery->rec.rows = numOfRes; + + if (numOfRes >= pQuery->rec.threshold) { + setQueryStatus(pQuery, QUERY_RESBUF_FULL); + } } - return 0; + return numOfRes; } void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY *tsCol, int32_t size, @@ -1449,8 +1437,7 @@ static void setWindowResultInfo(SResultInfo *pResultInfo, SQuery *pQuery, bool i } } -static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, SColumnModel *pTagsSchema, int16_t order, - bool isSTableQuery) { +static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, SColumnModel *pTagsSchema, int16_t order) { dTrace("QInfo:%p setup runtime env", GET_QINFO_ADDR(pRuntimeEnv)); SQuery *pQuery = pRuntimeEnv->pQuery; @@ -1464,7 +1451,6 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, SColumnModel pRuntimeEnv->offset[0] = 0; for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { SSqlFuncExprMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].pBase; -// SColIndexEx * pColIndexEx = &pSqlFuncMsg->colInfo; SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; pCtx->inputType = GET_COLUMN_TYPE(pQuery, i); @@ -1510,10 +1496,10 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, SColumnModel } // set the intermediate result output buffer - setWindowResultInfo(pRuntimeEnv->resultInfo, pQuery, isSTableQuery); + setWindowResultInfo(pRuntimeEnv->resultInfo, pQuery, pRuntimeEnv->stableQuery); // if it is group by normal column, do not set output buffer, the output buffer is pResult - if (!isGroupbyNormalCol(pQuery->pGroupbyExpr) && !isSTableQuery) { + if (!isGroupbyNormalCol(pQuery->pGroupbyExpr) && !pRuntimeEnv->stableQuery) { resetCtxOutputBuf(pRuntimeEnv); } @@ -1605,7 +1591,7 @@ bool isFixedOutputQuery(SQuery *pQuery) { // ignore the ts_comp function if (i == 0 && pExprMsg->functionId == TSDB_FUNC_PRJ && pExprMsg->numOfParams == 1 && - pExprMsg->colInfo.colIdx == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + pExprMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { continue; } @@ -1682,7 +1668,7 @@ static bool needReverseScan(SQuery *pQuery) { } ///////////////////////////////////////////////////////////////////////////////////////////// -void doGetAlignedIntervalQueryRangeImpl(SQuery *pQuery, int64_t key, int64_t keyFirst, int64_t keyLast, +void getAlignQueryTimeWindow(SQuery *pQuery, int64_t key, int64_t keyFirst, int64_t keyLast, int64_t *realSkey, int64_t *realEkey, STimeWindow *win) { assert(key >= keyFirst && key <= keyLast && pQuery->slidingTime <= pQuery->intervalTime); @@ -1834,9 +1820,9 @@ bool normalizeUnBoundLastRowQuery(SQInfo *pQInfo, SPointInterpoSupporter *pPoint static void setScanLimitationByResultBuffer(SQuery *pQuery) { if (isTopBottomQuery(pQuery)) { - pQuery->checkBufferInLoop = 0; + pQuery->checkBuffer = 0; } else if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { - pQuery->checkBufferInLoop = 0; + pQuery->checkBuffer = 0; } else { bool hasMultioutput = false; for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { @@ -1851,7 +1837,7 @@ static void setScanLimitationByResultBuffer(SQuery *pQuery) { } } - pQuery->checkBufferInLoop = hasMultioutput ? 1 : 0; + pQuery->checkBuffer = hasMultioutput ? 1 : 0; } } @@ -1901,9 +1887,9 @@ static void changeExecuteScanOrder(SQuery *pQuery, bool metricQuery) { // descending order query for last_row query if (isFirstLastRowQuery(pQuery)) { dTrace("QInfo:%p scan order changed for last_row query, old:%d, new:%d", GET_QINFO_ADDR(pQuery), - pQuery->order.order, TSQL_SO_DESC); + pQuery->order.order, TSDB_ORDER_DESC); - pQuery->order.order = TSQL_SO_DESC; + pQuery->order.order = TSDB_ORDER_DESC; int64_t skey = MIN(pQuery->window.skey, pQuery->window.ekey); int64_t ekey = MAX(pQuery->window.skey, pQuery->window.ekey); @@ -1916,56 +1902,56 @@ static void changeExecuteScanOrder(SQuery *pQuery, bool metricQuery) { if (isPointInterpoQuery(pQuery) && pQuery->intervalTime == 0) { if (!QUERY_IS_ASC_QUERY(pQuery)) { - dTrace(msg, GET_QINFO_ADDR(pQuery), "interp", pQuery->order.order, TSQL_SO_ASC, pQuery->window.skey, + dTrace(msg, GET_QINFO_ADDR(pQuery), "interp", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey, pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); } - pQuery->order.order = TSQL_SO_ASC; + pQuery->order.order = TSDB_ORDER_ASC; return; } if (pQuery->intervalTime == 0) { if (onlyFirstQuery(pQuery)) { if (!QUERY_IS_ASC_QUERY(pQuery)) { - dTrace(msg, GET_QINFO_ADDR(pQuery), "only-first", pQuery->order.order, TSQL_SO_ASC, pQuery->window.skey, + dTrace(msg, GET_QINFO_ADDR(pQuery), "only-first", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey, pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); } - pQuery->order.order = TSQL_SO_ASC; + pQuery->order.order = TSDB_ORDER_ASC; } else if (onlyLastQuery(pQuery)) { if (QUERY_IS_ASC_QUERY(pQuery)) { - dTrace(msg, GET_QINFO_ADDR(pQuery), "only-last", pQuery->order.order, TSQL_SO_DESC, pQuery->window.skey, + dTrace(msg, GET_QINFO_ADDR(pQuery), "only-last", pQuery->order.order, TSDB_ORDER_DESC, pQuery->window.skey, pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); } - pQuery->order.order = TSQL_SO_DESC; + pQuery->order.order = TSDB_ORDER_DESC; } } else { // interval query if (metricQuery) { if (onlyFirstQuery(pQuery)) { if (!QUERY_IS_ASC_QUERY(pQuery)) { - dTrace(msg, GET_QINFO_ADDR(pQuery), "only-first stable", pQuery->order.order, TSQL_SO_ASC, + dTrace(msg, GET_QINFO_ADDR(pQuery), "only-first stable", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey, pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); } - pQuery->order.order = TSQL_SO_ASC; + pQuery->order.order = TSDB_ORDER_ASC; } else if (onlyLastQuery(pQuery)) { if (QUERY_IS_ASC_QUERY(pQuery)) { - dTrace(msg, GET_QINFO_ADDR(pQuery), "only-last stable", pQuery->order.order, TSQL_SO_DESC, + dTrace(msg, GET_QINFO_ADDR(pQuery), "only-last stable", pQuery->order.order, TSDB_ORDER_DESC, pQuery->window.skey, pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); } - pQuery->order.order = TSQL_SO_DESC; + pQuery->order.order = TSDB_ORDER_DESC; } } } @@ -2103,8 +2089,7 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI continue; } - int32_t colInBuf = pQuery->pSelectExpr[i].pBase.colInfo.colIdxInBuf; - + int32_t colInBuf = 0;//pQuery->pSelectExpr[i].pBase.colInfo.colIdxInBuf; SInterpInfo *pInterpInfo = (SInterpInfo *)pRuntimeEnv->pCtx[i].aOutputBuf; pInterpInfo->pInterpDetail = calloc(1, sizeof(SInterpInfoDetail)); @@ -2275,12 +2260,12 @@ int32_t vnodeSTableQueryPrepare(SQInfo *pQInfo, SQuery *pQuery, void *param) { // set the ts-comp file traverse order if (param != NULL) { - int16_t order = (pQuery->order.order == pRuntimeEnv->pTSBuf->tsOrder) ? TSQL_SO_ASC : TSQL_SO_DESC; + int16_t order = (pQuery->order.order == pRuntimeEnv->pTSBuf->tsOrder) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; tsBufSetTraverseOrder(pRuntimeEnv->pTSBuf, order); } assert(0); - // int32_t ret = setupQueryRuntimeEnv(pMeter, pQuery, &pQInfo->runtimeEnv, pTagSchemaInfo, TSQL_SO_ASC, true); + // int32_t ret = setupQueryRuntimeEnv(pMeter, pQuery, &pQInfo->runtimeEnv, pTagSchemaInfo, TSDB_ORDER_ASC, true); // if (ret != TSDB_CODE_SUCCESS) { // return ret; // } @@ -2402,7 +2387,7 @@ static bool needToLoadDataBlock(SQuery *pQuery, SDataStatis *pDataStatis, SQLFun #if 0 for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; - int32_t colIndex = pFilterInfo->info.colIdx; + int32_t colIndex = pFilterInfo->info.colIndex; // this column not valid in current data block if (colIndex < 0 || pDataStatis[colIndex].colId != pFilterInfo->info.data.colId) { @@ -2480,7 +2465,7 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBl if (r == BLK_DATA_NO_NEEDED) { qTrace("QInfo:%p slot:%d, data block ignored, brange:%" PRId64 "-%" PRId64 ", rows:%d", - GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->size); + GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); } else if (r == BLK_DATA_FILEDS_NEEDED) { if (tsdbRetrieveDataBlockStatisInfo(pRuntimeEnv->pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { // return DISK_DATA_LOAD_FAILED; @@ -2500,7 +2485,7 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBl * filter the data block according to the value filter condition. * no need to load the data block, continue for next block */ - if (!needToLoadDataBlock(pQuery, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->size)) { + if (!needToLoadDataBlock(pQuery, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->rows)) { #if defined(_DEBUG_VIEW) dTrace("QInfo:%p fileId:%d, slot:%d, block discarded by per-filter", GET_QINFO_ADDR(pQuery), pQuery->fileId, pQuery->slot); @@ -2576,8 +2561,6 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - int64_t cnt = 0; - dTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", lastkey:%" PRId64 ", order:%d", GET_QINFO_ADDR(pRuntimeEnv), pQuery->window.skey, pQuery->window.ekey, pQuery->lastKey, pQuery->order.order); @@ -2585,43 +2568,66 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { while (tsdbNextDataBlock(pQueryHandle)) { // check if query is killed or not set the status of query to pass the status check if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { - return cnt; + return 0; } SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle); + // todo extract methods if (isIntervalQuery(pQuery) && pRuntimeEnv->windowResInfo.prevSKey == 0) { - TSKEY skey1, ekey1; - STimeWindow w = {0}; + TSKEY skey1, ekey1; + STimeWindow w = {0}; SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; if (QUERY_IS_ASC_QUERY(pQuery)) { - doGetAlignedIntervalQueryRangeImpl(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, + getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &skey1, &ekey1, &w); pWindowResInfo->startTime = w.skey; pWindowResInfo->prevSKey = w.skey; } else { // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp - TSKEY winStart = blockInfo.window.ekey - pQuery->intervalTime; - doGetAlignedIntervalQueryRangeImpl(pQuery, winStart, pQuery->window.ekey, blockInfo.window.ekey, &skey1, &ekey1, - &w); + TSKEY start = blockInfo.window.ekey - pQuery->intervalTime; + getAlignQueryTimeWindow(pQuery, start, pQuery->window.ekey, blockInfo.window.ekey, &skey1, &ekey1, &w); pWindowResInfo->startTime = pQuery->window.skey; pWindowResInfo->prevSKey = w.skey; } } + + // in case of prj/diff query, ensure the output buffer is sufficient to accomodate the results of current block + if (!isIntervalQuery(pQuery) && !isGroupbyNormalCol(pQuery->pGroupbyExpr) && !isFixedOutputQuery(pQuery)) { + SResultRec* pRec = &pQuery->rec; + + if (pQuery->rec.capacity - pQuery->rec.rows < blockInfo.rows) { + int32_t remain = pRec->capacity - pRec->rows; + int32_t newSize = pRec->capacity + (blockInfo.rows - remain); + + for(int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t bytes = pQuery->pSelectExpr[i].resBytes; + + char* tmp = realloc(pQuery->sdata[i], bytes * newSize + sizeof(SData)); + if (tmp == NULL) { // todo handle the oom + } else { + pQuery->sdata[i] = (SData*) tmp; + } + + // set the pCtx output buffer position + pRuntimeEnv->pCtx[i].aOutputBuf = pQuery->sdata[i]->data + pRec->rows*bytes; + } + + pRec->capacity = newSize; + } + } - int32_t numOfRes = 0; SDataStatis *pStatis = NULL; SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); - int32_t forwardStep = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, &numOfRes, + int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, &pRuntimeEnv->windowResInfo, pDataBlock); - dTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d", - GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.size); + dTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, res:%d", + GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes); // save last access position - cnt += forwardStep; if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { break; } @@ -2644,7 +2650,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { } } - return cnt; + return 0; } static void updatelastkey(SQuery *pQuery, STableQueryInfo *pTableQInfo) { pTableQInfo->lastKey = pQuery->lastKey; } @@ -2682,19 +2688,19 @@ void vnodeSetTagValueInParam(tSidSet *pSidSet, SQueryRuntimeEnv *pRuntimeEnv, vo } else { // set tag value, by which the results are aggregated. for (int32_t idx = 0; idx < pQuery->numOfOutputCols; ++idx) { - SColIndexEx *pColEx = &pQuery->pSelectExpr[idx].pBase.colInfo; + SColIndex *pColEx = &pQuery->pSelectExpr[idx].pBase.colInfo; // ts_comp column required the tag value for join filter if (!TSDB_COL_IS_TAG(pColEx->flag)) { continue; } - doSetTagValueInParam(pTagSchema, pColEx->colIdx, pMeterSidInfo, &pRuntimeEnv->pCtx[idx].tag); + doSetTagValueInParam(pTagSchema, pColEx->colIndex, pMeterSidInfo, &pRuntimeEnv->pCtx[idx].tag); } // set the join tag for first column SSqlFuncExprMsg *pFuncMsg = &pQuery->pSelectExpr[0].pBase; - if (pFuncMsg->functionId == TSDB_FUNC_TS && pFuncMsg->colInfo.colIdx == PRIMARYKEY_TIMESTAMP_COL_INDEX && + if (pFuncMsg->functionId == TSDB_FUNC_TS && pFuncMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX && pRuntimeEnv->pTSBuf != NULL) { assert(pFuncMsg->numOfParams == 1); doSetTagValueInParam(pTagSchema, pFuncMsg->arg->argValue.i64, pMeterSidInfo, &pRuntimeEnv->pCtx[0].tag); @@ -2821,13 +2827,13 @@ void UNUSED_FUNC displayInterResult(SData **pdata, SQuery *pQuery, int32_t numOf for (int32_t i = 0; i < numOfCols; ++i) { switch (pQuery->pSelectExpr[i].resType) { case TSDB_DATA_TYPE_BINARY: { - int32_t colIdx = pQuery->pSelectExpr[i].pBase.colInfo.colIdx; + int32_t colIndex = pQuery->pSelectExpr[i].pBase.colInfo.colIndex; int32_t type = 0; if (TSDB_COL_IS_TAG(pQuery->pSelectExpr[i].pBase.colInfo.flag)) { type = pQuery->pSelectExpr[i].resType; } else { - type = pMeterObj->schema[colIdx].type; + type = pMeterObj->schema[colIndex].type; } printBinaryData(pQuery->pSelectExpr[i].pBase.functionId, pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j, type); @@ -2978,9 +2984,9 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { offset += pData->numOfElems; } - assert(pQuery->rec.size == 0); + assert(pQuery->rec.rows == 0); - pQuery->rec.size += rows; + pQuery->rec.rows += rows; pQInfo->offset += 1; } @@ -3206,8 +3212,8 @@ static void doDisableFunctsForSupplementaryScan(SQuery *pQuery, SWindowResInfo * for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { int32_t functId = pQuery->pSelectExpr[j].pBase.functionId; - if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSQL_SO_DESC) || - ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSQL_SO_ASC)) { + if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_DESC) || + ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSDB_ORDER_ASC)) { buf->resultInfo[j].complete = false; } else if (functId != TSDB_FUNC_TS && functId != TSDB_FUNC_TAG) { buf->resultInfo[j].complete = true; @@ -3232,8 +3238,8 @@ void disableFunctForTableSuppleScan(SQueryRuntimeEnv *pRuntimeEnv, int32_t order int32_t functId = pQuery->pSelectExpr[j].pBase.functionId; SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[j]; - if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSQL_SO_DESC) || - ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSQL_SO_ASC)) { + if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_DESC) || + ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSDB_ORDER_ASC)) { pCtx->resultInfo->complete = false; } else if (functId != TSDB_FUNC_TS && functId != TSDB_FUNC_TAG) { pCtx->resultInfo->complete = true; @@ -3309,7 +3315,7 @@ void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; } - memset(pQuery->sdata[i]->data, 0, (size_t)pQuery->pSelectExpr[i].resBytes * pQuery->rec.capacity); + memset(pQuery->sdata[i]->data, 0, (size_t) pQuery->pSelectExpr[i].resBytes * pQuery->rec.capacity); } initCtxOutputBuf(pRuntimeEnv); @@ -3348,22 +3354,22 @@ void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; + pRuntimeEnv->pCtx[j].currentStage = 0; - aAggs[functionId].init(&pRuntimeEnv->pCtx[j]); } } void doSkipResults(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - if (pQuery->rec.size == 0 || pQuery->limit.offset == 0) { + if (pQuery->rec.rows == 0 || pQuery->limit.offset == 0) { return; } - if (pQuery->rec.size <= pQuery->limit.offset) { - pQuery->limit.offset -= pQuery->rec.size; + if (pQuery->rec.rows <= pQuery->limit.offset) { + pQuery->limit.offset -= pQuery->rec.rows; - pQuery->rec.size = 0; + pQuery->rec.rows = 0; // pQuery->pointsOffset = pQuery->rec.pointsToRead; // clear all data in result buffer resetCtxOutputBuf(pRuntimeEnv); @@ -3372,7 +3378,7 @@ void doSkipResults(SQueryRuntimeEnv *pRuntimeEnv) { pQuery->status &= (~QUERY_RESBUF_FULL); } else { int32_t numOfSkip = (int32_t)pQuery->limit.offset; - pQuery->rec.size -= numOfSkip; + pQuery->rec.rows -= numOfSkip; for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; @@ -3527,6 +3533,7 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { doScanAllDataBlocks(pRuntimeEnv); if (!needScanDataBlocksAgain(pRuntimeEnv)) { + // restore the status if (pRuntimeEnv->scanFlag == REPEAT_SCAN) { pQuery->status = status; @@ -3546,7 +3553,6 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { /* check if query is killed or not */ if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { -// setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); return; } } @@ -3677,7 +3683,7 @@ void restoreIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *p /** * set output buffer for different group * @param pRuntimeEnv - * @param pDataBlockInfoEx + * @param pDataBlockInfo */ void setExecutionContext(SQInfo *pQInfo, STableQueryInfo *pTableQueryInfo, int32_t meterIdx, int32_t groupIdx, TSKEY nextKey) { @@ -3792,7 +3798,7 @@ void setIntervalQueryRange(STableQueryInfo *pTableQueryInfo, SQInfo *pQInfo, TSK STimeWindow w = {0}; SWindowResInfo *pWindowResInfo = &pTableQueryInfo->windowResInfo; - doGetAlignedIntervalQueryRangeImpl(pQuery, win.skey, win.skey, win.ekey, &skey1, &ekey1, &w); + getAlignQueryTimeWindow(pQuery, win.skey, win.skey, win.ekey, &skey1, &ekey1, &w); pWindowResInfo->startTime = pQuery->window.skey; // windowSKey may be 0 in case of 1970 timestamp if (pWindowResInfo->prevSKey == 0) { @@ -3862,7 +3868,7 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orde dTrace("QInfo:%p start to copy data from windowResInfo to pQuery buf", GET_QINFO_ADDR(pQuery)); int32_t totalSubset = getNumOfSubset(pQInfo); - if (orderType == TSQL_SO_ASC) { + if (orderType == TSDB_ORDER_ASC) { startIdx = pQInfo->subgroupIdx; step = 1; } else { // desc order copy all data @@ -3928,12 +3934,12 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orde void copyFromWindowResToSData(SQInfo *pQInfo, SWindowResult *result) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - int32_t orderType = (pQuery->pGroupbyExpr != NULL) ? pQuery->pGroupbyExpr->orderType : TSQL_SO_ASC; + int32_t orderType = (pQuery->pGroupbyExpr != NULL) ? pQuery->pGroupbyExpr->orderType : TSDB_ORDER_ASC; int32_t numOfResult = doCopyToSData(pQInfo, result, orderType); - pQuery->rec.size += numOfResult; + pQuery->rec.rows += numOfResult; - assert(pQuery->rec.size <= pQuery->rec.capacity); + assert(pQuery->rec.rows <= pQuery->rec.capacity); } static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv, STableDataInfo *pTableDataInfo) { @@ -3957,15 +3963,12 @@ void stableApplyFunctionsOnBlock(SQueryRuntimeEnv* pRuntimeEnv, STableDataInfo * STableQueryInfo * pTableQueryInfo = pTableDataInfo->pTableQInfo; SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; - int32_t numOfRes = 0; if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) { // numOfRes = rowwiseApplyAllFunctions(pRuntimeEnv, &forwardStep, pFields, pDataBlockInfo, pWindowResInfo); } else { - numOfRes = blockwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); + blockwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); } - assert(numOfRes >= 0); - updateWindowResNumOfRes(pRuntimeEnv, pTableDataInfo); updatelastkey(pQuery, pTableQueryInfo); } @@ -3981,7 +3984,7 @@ bool vnodeHasRemainResults(void *handle) { SQuery * pQuery = pRuntimeEnv->pQuery; SInterpolationInfo *pInterpoInfo = &pRuntimeEnv->interpoInfo; - if (pQuery->limit.limit > 0 && pQuery->rec.size >= pQuery->limit.limit) { + if (pQuery->limit.limit > 0 && pQuery->rec.rows >= pQuery->limit.limit) { return false; } @@ -4152,7 +4155,7 @@ void vnodePrintQueryStatistics(SQInfo *pQInfo) { #endif } -int32_t doInitializeQInfo(SQInfo *pQInfo, void *param, void* tsdb, bool isSTableQuery) { +int32_t doInitQInfo(SQInfo *pQInfo, void *param, void* tsdb, bool isSTableQuery) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; @@ -4181,13 +4184,15 @@ int32_t doInitializeQInfo(SQInfo *pQInfo, void *param, void* tsdb, bool isSTable pRuntimeEnv->pQuery = pQuery; pRuntimeEnv->pTSBuf = param; pRuntimeEnv->cur.vnodeIndex = -1; + pRuntimeEnv->stableQuery = isSTableQuery; + if (param != NULL) { - int16_t order = (pQuery->order.order == pRuntimeEnv->pTSBuf->tsOrder) ? TSQL_SO_ASC : TSQL_SO_DESC; + int16_t order = (pQuery->order.order == pRuntimeEnv->pTSBuf->tsOrder) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; tsBufSetTraverseOrder(pRuntimeEnv->pTSBuf, order); } // create runtime environment - code = setupQueryRuntimeEnv(pRuntimeEnv, NULL, pQuery->order.order, isSTableQuery); + code = setupQueryRuntimeEnv(pRuntimeEnv, NULL, pQuery->order.order); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -4270,8 +4275,6 @@ int32_t doInitializeQInfo(SQInfo *pQInfo, void *param, void* tsdb, bool isSTable // the pQuery->window.skey is changed during normalizedFirstQueryRange, so set the newest lastkey value pQuery->lastKey = pQuery->window.skey; - pRuntimeEnv->stableQuery = false; - return TSDB_CODE_SUCCESS; } @@ -4281,7 +4284,7 @@ static UNUSED_FUNC bool isGroupbyEachTable(SSqlGroupbyExpr *pGroupbyExpr, tSidSe } for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { - SColIndexEx *pColIndex = &pGroupbyExpr->columnInfo[i]; + SColIndex *pColIndex = &pGroupbyExpr->columnInfo[i]; if (pColIndex->flag == TSDB_COL_TAG) { // assert(pSidset->numOfTables == pSidset->numOfSubSet); return true; @@ -4454,7 +4457,7 @@ static UNUSED_FUNC int64_t doCheckMetersInGroup(SQInfo *pQInfo, int32_t index, i // accumulate the point interpolation result if (numOfRes > 0) { - pQuery->rec.size += numOfRes; + pQuery->rec.rows += numOfRes; forwardCtxOutputBuf(pRuntimeEnv, numOfRes); } @@ -4821,13 +4824,13 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); } - pQuery->rec.size += pQuery->rec.size; + pQuery->rec.rows += pQuery->rec.rows; - if (pQuery->rec.size == 0) { + if (pQuery->rec.rows == 0) { // vnodePrintQueryStatistics(pSupporter); } - dTrace("QInfo:%p current:%lldd, total:%lldd", pQInfo, pQuery->rec.size, pQuery->rec.total); + dTrace("QInfo:%p current:%lldd, total:%lldd", pQInfo, pQuery->rec.rows, pQuery->rec.total); return; } @@ -4884,7 +4887,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { } // handle the limitation of output buffer - dTrace("QInfo:%p points returned:%d, total:%d", pQInfo, pQuery->rec.size, pQuery->rec.total); + dTrace("QInfo:%p points returned:%d, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total); } /* @@ -4905,7 +4908,7 @@ static void tableFixedOutputProcessor(SQInfo *pQInfo) { } // since the numOfOutputElems must be identical for all sql functions that are allowed to be executed simutanelously. - pQuery->rec.size = getNumOfResult(pRuntimeEnv); + pQuery->rec.rows = getNumOfResult(pRuntimeEnv); // assert(pQuery->size <= pQuery->pointsToRead && // Q_STATUS_EQUAL(pQuery->over, QUERY_COMPLETED)); @@ -4917,12 +4920,12 @@ static void tableFixedOutputProcessor(SQInfo *pQInfo) { doSkipResults(pRuntimeEnv); doRevisedResultsByLimit(pQInfo); - pQuery->rec.size = pQuery->rec.size; + pQuery->rec.rows = pQuery->rec.rows; } -static void tableMultiOutputProcessor(SQInfo *pQInfo) { +static void tableMultiOutputProcess(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; + SQuery *pQuery = pRuntimeEnv->pQuery; // for ts_comp query, re-initialized is not allowed if (!isTSCompQuery(pQuery)) { @@ -4937,8 +4940,8 @@ static void tableMultiOutputProcessor(SQInfo *pQInfo) { return; } - pQuery->rec.size = getNumOfResult(pRuntimeEnv); - if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols > 0 && pQuery->rec.size > 0) { + pQuery->rec.rows = getNumOfResult(pRuntimeEnv); + if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols > 0 && pQuery->rec.rows > 0) { doSkipResults(pRuntimeEnv); } @@ -4946,7 +4949,7 @@ static void tableMultiOutputProcessor(SQInfo *pQInfo) { * 1. if pQuery->size == 0, pQuery->limit.offset >= 0, still need to check data * 2. if pQuery->size > 0, pQuery->limit.offset must be 0 */ - if (pQuery->rec.size > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { + if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { break; } @@ -4958,17 +4961,16 @@ static void tableMultiOutputProcessor(SQInfo *pQInfo) { doRevisedResultsByLimit(pQInfo); if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { - dTrace("QInfo:%p query paused due to buffer limitation, next qrange:%" PRId64 "-%" PRId64, - pQInfo, pQuery->lastKey, pQuery->window.ekey); + dTrace("QInfo:%p query paused due to output limitation, next qrange:%" PRId64 "-%" PRId64, + pQInfo, pQuery->lastKey, pQuery->window.ekey); } // dTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned, totalRead:%d totalReturn:%d", pQInfo, pMeterObj->vnode, // pMeterObj->sid, pMeterObj->meterId, pQuery->size, pQInfo->size, pQInfo->pointsReturned); -// pQuery->pointsOffset = pQuery->pointsToRead; //restore the available buffer -// if (!isTSCompQuery(pQuery)) { -// assert(pQuery->size <= pQuery->pointsToRead); -// } + if (!isTSCompQuery(pQuery)) { + assert(pQuery->rec.rows <= pQuery->rec.capacity); + } } static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv) { @@ -5015,7 +5017,7 @@ static void tableIntervalProcessor(SQInfo *pQInfo) { if (isIntervalQuery(pQuery)) { pQInfo->subgroupIdx = 0; // always start from 0 - pQuery->rec.size = 0; + pQuery->rec.rows = 0; copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); clearFirstNTimeWindow(pRuntimeEnv, pQInfo->subgroupIdx); @@ -5026,37 +5028,37 @@ static void tableIntervalProcessor(SQInfo *pQInfo) { doRevisedResultsByLimit(pQInfo); break; } else { - taosInterpoSetStartInfo(&pRuntimeEnv->interpoInfo, pQuery->rec.size, pQuery->interpoType); + taosInterpoSetStartInfo(&pRuntimeEnv->interpoInfo, pQuery->rec.rows, pQuery->interpoType); SData **pInterpoBuf = pRuntimeEnv->pInterpoBuf; for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - memcpy(pInterpoBuf[i]->data, pQuery->sdata[i]->data, pQuery->rec.size * pQuery->pSelectExpr[i].resBytes); + memcpy(pInterpoBuf[i]->data, pQuery->sdata[i]->data, pQuery->rec.rows * pQuery->pSelectExpr[i].resBytes); } numOfInterpo = 0; - pQuery->rec.size = vnodeQueryResultInterpolate( - pQInfo, (tFilePage **)pQuery->sdata, (tFilePage **)pInterpoBuf, pQuery->rec.size, &numOfInterpo); + pQuery->rec.rows = vnodeQueryResultInterpolate( + pQInfo, (tFilePage **)pQuery->sdata, (tFilePage **)pInterpoBuf, pQuery->rec.rows, &numOfInterpo); - dTrace("QInfo: %p interpo completed, final:%d", pQInfo, pQuery->rec.size); - if (pQuery->rec.size > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { + dTrace("QInfo: %p interpo completed, final:%d", pQInfo, pQuery->rec.rows); + if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { doRevisedResultsByLimit(pQInfo); break; } // no result generated yet, continue retrieve data - pQuery->rec.size = 0; + pQuery->rec.rows = 0; } } // all data scanned, the group by normal column can return if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // todo refactor with merge interval time result pQInfo->subgroupIdx = 0; - pQuery->rec.size = 0; + pQuery->rec.rows = 0; copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); clearFirstNTimeWindow(pRuntimeEnv, pQInfo->subgroupIdx); } - pQuery->rec.size += pQuery->rec.size; + pQuery->rec.rows += pQuery->rec.rows; pQInfo->pointsInterpo += numOfInterpo; // dTrace("%p vid:%d sid:%d id:%s, %d points returned %d points interpo, totalRead:%d totalInterpo:%d @@ -5065,7 +5067,7 @@ static void tableIntervalProcessor(SQInfo *pQInfo) { // pQInfo->size - pQInfo->pointsInterpo, pQInfo->pointsInterpo, pQInfo->pointsReturned); } -static void singleTableQueryImpl(SQInfo* pQInfo) { +static void tableQueryImpl(SQInfo* pQInfo) { SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; SQuery* pQuery = pRuntimeEnv->pQuery; @@ -5076,15 +5078,15 @@ static void singleTableQueryImpl(SQInfo* pQInfo) { */ int32_t numOfInterpo = 0; int32_t remain = taosNumOfRemainPoints(&pRuntimeEnv->interpoInfo); - pQuery->rec.size = vnodeQueryResultInterpolate(pQInfo, (tFilePage **)pQuery->sdata, + pQuery->rec.rows = vnodeQueryResultInterpolate(pQInfo, (tFilePage **)pQuery->sdata, (tFilePage **)pRuntimeEnv->pInterpoBuf, remain, &numOfInterpo); doRevisedResultsByLimit(pQInfo); pQInfo->pointsInterpo += numOfInterpo; - pQuery->rec.size += pQuery->rec.size; + pQuery->rec.rows += pQuery->rec.rows; - dTrace("QInfo:%p current:%d returned, total:%d", pQInfo, pQuery->rec.size, pQuery->rec.total); + dTrace("QInfo:%p current:%d returned, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total); sem_post(&pQInfo->dataReady); return; } @@ -5096,17 +5098,17 @@ static void singleTableQueryImpl(SQInfo* pQInfo) { ((isIntervalQuery(pQuery) && pQuery->rec.total < pQuery->limit.limit))) { // todo limit the output for interval query? - pQuery->rec.size = 0; + pQuery->rec.rows = 0; pQInfo->subgroupIdx = 0; // always start from 0 if (pRuntimeEnv->windowResInfo.size > 0) { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); - pQuery->rec.size += pQuery->rec.size; + pQuery->rec.rows += pQuery->rec.rows; clearFirstNTimeWindow(pRuntimeEnv, pQInfo->subgroupIdx); - if (pQuery->rec.size > 0) { - dTrace("QInfo:%p %d rows returned from group results, total:%d", pQInfo, pQuery->rec.size, pQuery->rec.total); + if (pQuery->rec.rows > 0) { + dTrace("QInfo:%p %d rows returned from group results, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total); sem_post(&pQInfo->dataReady); return; } @@ -5120,23 +5122,21 @@ static void singleTableQueryImpl(SQInfo* pQInfo) { } // number of points returned during this query - pQuery->rec.size = 0; + pQuery->rec.rows = 0; int64_t st = taosGetTimestampUs(); // group by normal column, sliding window query, interval query are handled by interval query processor if (isIntervalQuery(pQuery) || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // interval (down sampling operation) tableIntervalProcessor(pQInfo); - } else { - if (isFixedOutputQuery(pQuery)) { - assert(pQuery->checkBufferInLoop == 0); - - tableFixedOutputProcessor(pQInfo); - } else { // diff/add/multiply/subtract/division - assert(pQuery->checkBufferInLoop == 1); - tableMultiOutputProcessor(pQInfo); - } + } else if (isFixedOutputQuery(pQuery)) { + assert(pQuery->checkBuffer == 0); + tableFixedOutputProcessor(pQInfo); + } else { // diff/add/multiply/subtract/division + assert(pQuery->checkBuffer == 1); + tableMultiOutputProcess(pQInfo); } + // record the total elapsed time pQInfo->elapsedTime += (taosGetTimestampUs() - st); @@ -5144,16 +5144,16 @@ static void singleTableQueryImpl(SQInfo* pQInfo) { if (isQueryKilled(pQInfo)) { dTrace("QInfo:%p query is killed", pQInfo); } else { - dTrace("QInfo:%p query task completed, %" PRId64 " rows will returned, total:%" PRId64 " rows", pQInfo, pQuery->rec.size, + dTrace("QInfo:%p query task completed, %" PRId64 " rows will returned, total:%" PRId64 " rows", pQInfo, pQuery->rec.rows, pQuery->rec.total); } sem_post(&pQInfo->dataReady); } -static void multiTableQueryImpl(SQInfo* pQInfo) { +static void stableQueryImpl(SQInfo* pQInfo) { SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - pQuery->rec.size = 0; + pQuery->rec.rows = 0; int64_t st = taosGetTimestampUs(); @@ -5161,7 +5161,7 @@ static void multiTableQueryImpl(SQInfo* pQInfo) { (isFixedOutputQuery(pQuery) && (!isPointInterpoQuery(pQuery)) && !isGroupbyNormalCol(pQuery->pGroupbyExpr))) { multiTableQueryProcess(pQInfo); } else { - assert((pQuery->checkBufferInLoop == 1 && pQuery->intervalTime == 0) || isPointInterpoQuery(pQuery) || + assert((pQuery->checkBuffer == 1 && pQuery->intervalTime == 0) || isPointInterpoQuery(pQuery) || isGroupbyNormalCol(pQuery->pGroupbyExpr)); vnodeSTableSeqProcessor(pQInfo); @@ -5171,7 +5171,7 @@ static void multiTableQueryImpl(SQInfo* pQInfo) { pQInfo->elapsedTime += (taosGetTimestampUs() - st); // taosInterpoSetStartInfo(&pQInfo->runtimeEnv.interpoInfo, pQuery->size, pQInfo->query.interpoType); - if (pQuery->rec.size == 0) { + if (pQuery->rec.rows == 0) { int32_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); dTrace("QInfo:%p over, %d tables queried, %d points are returned", pQInfo, numOfTables, pQuery->rec.total); // vnodePrintQueryStatistics(pSupporter); @@ -5266,7 +5266,7 @@ static char* createTableIdList(SQueryTableMsg* pQueryMsg, char* pMsg, SArray** p * @return */ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncExprMsg ***pExpr, - wchar_t** tagCond) { + char** tagCond) { pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); @@ -5284,7 +5284,6 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->numOfOutputCols = htons(pQueryMsg->numOfOutputCols); pQueryMsg->numOfGroupCols = htons(pQueryMsg->numOfGroupCols); pQueryMsg->tagCondLen = htons(pQueryMsg->tagCondLen); - pQueryMsg->nameCondLen = htons(pQueryMsg->nameCondLen); pQueryMsg->tsOffset = htonl(pQueryMsg->tsOffset); pQueryMsg->tsLen = htonl(pQueryMsg->tsLen); pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks); @@ -5344,7 +5343,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { (*pExpr)[i] = pExprMsg; - pExprMsg->colInfo.colIdx = htons(pExprMsg->colInfo.colIdx); + pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); pExprMsg->functionId = htons(pExprMsg->functionId); @@ -5398,7 +5397,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->orderByIdx = htons(pQueryMsg->orderByIdx); pQueryMsg->orderType = htons(pQueryMsg->orderType); - pMsg += sizeof(SColIndexEx) * pQueryMsg->numOfGroupCols; + pMsg += sizeof(SColIndex) * pQueryMsg->numOfGroupCols; } else { pQueryMsg->groupbyTagIds = 0; } @@ -5417,8 +5416,8 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, // the tag query condition expression string is located at the end of query msg if (pQueryMsg->tagCondLen > 0) { - *tagCond = calloc(1, pQueryMsg->tagCondLen * TSDB_NCHAR_SIZE); - memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen * TSDB_NCHAR_SIZE); + *tagCond = calloc(1, pQueryMsg->tagCondLen); + memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen); } dTrace("qmsg:%p query on %d table(s), qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, ts order:%d, " @@ -5468,10 +5467,10 @@ static int32_t buildAirthmeticExprFromMsg(SSqlFunctionExpr *pExpr, SQueryTableMs // there may be duplicated referenced columns. num = i + 1; - pBinaryExprInfo->pReqColumns = malloc(sizeof(SColIndexEx) * num); + pBinaryExprInfo->pReqColumns = malloc(sizeof(SColIndex) * num); for (int32_t k = 0; k < num; ++k) { - SColIndexEx* pColIndex = &pBinaryExprInfo->pReqColumns[k]; + SColIndex* pColIndex = &pBinaryExprInfo->pReqColumns[k]; pColIndex->colId = ids[k]; } @@ -5568,19 +5567,19 @@ static SSqlGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, int3 // using group by tag columns SSqlGroupbyExpr *pGroupbyExpr = - (SSqlGroupbyExpr *)malloc(sizeof(SSqlGroupbyExpr) + pQueryMsg->numOfGroupCols * sizeof(SColIndexEx)); + (SSqlGroupbyExpr *)malloc(sizeof(SSqlGroupbyExpr) + pQueryMsg->numOfGroupCols * sizeof(SColIndex)); if (pGroupbyExpr == NULL) { *code = TSDB_CODE_SERV_OUT_OF_MEMORY; return NULL; } - SColIndexEx *pGroupbyColInfo = (SColIndexEx *)pQueryMsg->groupbyTagIds; + SColIndex *pGroupbyColInfo = (SColIndex *)pQueryMsg->groupbyTagIds; pGroupbyExpr->numOfGroupCols = pQueryMsg->numOfGroupCols; pGroupbyExpr->orderType = pQueryMsg->orderType; pGroupbyExpr->orderIndex = pQueryMsg->orderByIdx; - memcpy(pGroupbyExpr->columnInfo, pGroupbyColInfo, sizeof(SColIndexEx) * pGroupbyExpr->numOfGroupCols); + memcpy(pGroupbyExpr->columnInfo, pGroupbyColInfo, sizeof(SColIndex) * pGroupbyExpr->numOfGroupCols); return pGroupbyExpr; } @@ -5630,9 +5629,9 @@ static int32_t vnodeCreateFilterInfo(void *pQInfo, SQuery *pQuery) { return TSDB_CODE_INVALID_QUERY_MSG; } - if ((lower == TSDB_RELATION_LARGE_EQUAL || lower == TSDB_RELATION_LARGE) && + if ((lower == TSDB_RELATION_GREATER_EQUAL || lower == TSDB_RELATION_GREATER) && (upper == TSDB_RELATION_LESS_EQUAL || upper == TSDB_RELATION_LESS)) { - if (lower == TSDB_RELATION_LARGE_EQUAL) { + if (lower == TSDB_RELATION_GREATER_EQUAL) { if (upper == TSDB_RELATION_LESS_EQUAL) { pSingleColFilter->fp = rangeFilterArray[4]; } else { @@ -5673,16 +5672,16 @@ static void doUpdateExprColumnIndex(SQuery* pQuery) { // int32_t i = 0, j = 0; // while (i < pQuery->numOfCols && j < pMeterObj->numOfColumns) { // if (pQuery->colList[i].data.colId == pMeterObj->schema[j].colId) { -// pQuery->colList[i++].colIdx = (int16_t)j++; +// pQuery->colList[i++].colIndex = (int16_t)j++; // } else if (pQuery->colList[i].data.colId < pMeterObj->schema[j].colId) { -// pQuery->colList[i++].colIdx = -1; +// pQuery->colList[i++].colIndex = -1; // } else if (pQuery->colList[i].data.colId > pMeterObj->schema[j].colId) { // j++; // } // } // while (i < pQuery->numOfCols) { -// pQuery->colList[i++].colIdx = -1; // not such column in current meter +// pQuery->colList[i++].colIndex = -1; // not such column in current meter // } for(int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { @@ -5691,10 +5690,10 @@ static void doUpdateExprColumnIndex(SQuery* pQuery) { continue; } - SColIndexEx* pColIndexEx = &pSqlExprMsg->colInfo; + SColIndex* pColIndexEx = &pSqlExprMsg->colInfo; for(int32_t f = 0; f < pQuery->numOfCols; ++f) { if (pColIndexEx->colId == pQuery->colList[f].info.colId) { - pColIndexEx->colIdx = f; + pColIndexEx->colIndex = f; break; } } @@ -5775,7 +5774,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou // set the output buffer capacity pQuery->rec.capacity = 4096; - pQuery->rec.threshold = 2; + pQuery->rec.threshold = 4000; for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { assert(pExprs[col].interResBytes >= pExprs[col].resBytes); @@ -5855,7 +5854,7 @@ static bool isValidQInfo(void *param) { } static void freeQInfo(SQInfo *pQInfo); -static int32_t initializeQInfo(SQueryTableMsg *pQueryMsg, void* tsdb, SQInfo *pQInfo, bool isSTable) { +static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void* tsdb, SQInfo *pQInfo, bool isSTable) { int32_t code = TSDB_CODE_SUCCESS; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; @@ -5880,7 +5879,7 @@ static int32_t initializeQInfo(SQueryTableMsg *pQueryMsg, void* tsdb, SQInfo *pQ } // filter the qualified - if ((code = doInitializeQInfo(pQInfo, pTSBuf, tsdb, isSTable)) != TSDB_CODE_SUCCESS) { + if ((code = doInitQInfo(pQInfo, pTSBuf, tsdb, isSTable)) != TSDB_CODE_SUCCESS) { goto _error; } @@ -6004,11 +6003,11 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { pQuery->sdata[0]->data, strerror(errno)); } } else { - doCopyQueryResultToMsg(pQInfo, pQuery->rec.size, data); + doCopyQueryResultToMsg(pQInfo, pQuery->rec.rows, data); } - pQuery->rec.total += pQuery->rec.size; - dTrace("QInfo:%p current:%d, total:%d", pQInfo, pQuery->rec.size, pQuery->rec.total); + pQuery->rec.total += pQuery->rec.rows; + dTrace("QInfo:%p current:%d, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total); return TSDB_CODE_SUCCESS; @@ -6020,9 +6019,9 @@ int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryMsg, SQInfo **pQInfo) int32_t code = TSDB_CODE_SUCCESS; + char* tagCond = NULL; SArray *pTableIdList = NULL; SSqlFuncExprMsg** pExprMsg = NULL; - wchar_t* tagCond = NULL; if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond)) != TSDB_CODE_SUCCESS) { return code; } @@ -6050,21 +6049,23 @@ int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryMsg, SQInfo **pQInfo) goto _query_over; } - // super table query - SArray* res = NULL; bool isSTableQuery = false; + SArray* res = NULL; if ((pQueryMsg->queryType & TSDB_QUERY_TYPE_STABLE_QUERY) != 0) { isSTableQuery = true; STableId* id = taosArrayGet(pTableIdList, 0); id->uid = -1; - res = tsdbQueryTableList(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen); + res = taosArrayInit(8, sizeof(STableId)); + + /*int32_t ret =*/ tsdbQueryTags(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, res); if (taosArrayGetSize(res) == 0) { // no qualified table in stable query in this vnode code = TSDB_CODE_SUCCESS; goto _query_over; } } else { + assert(taosArrayGetSize(pTableIdList) == 1); res = pTableIdList; } @@ -6073,7 +6074,7 @@ int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryMsg, SQInfo **pQInfo) code = TSDB_CODE_SERV_OUT_OF_MEMORY; } - code = initializeQInfo(pQueryMsg, tsdb, *pQInfo, isSTableQuery); + code = initQInfo(pQueryMsg, tsdb, *pQInfo, isSTableQuery); _query_over: if (code != TSDB_CODE_SUCCESS) { @@ -6116,11 +6117,10 @@ void qTableQuery(SQInfo *pQInfo) { dTrace("QInfo:%p query task is launched", pQInfo); - int32_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); - if (numOfTables == 1) { - singleTableQueryImpl(pQInfo); + if (pQInfo->runtimeEnv.stableQuery) { + stableQueryImpl(pQInfo); } else { - multiTableQueryImpl(pQInfo); + tableQueryImpl(pQInfo); } // vnodeDecRefCount(pQInfo); @@ -6138,7 +6138,7 @@ int32_t qRetrieveQueryResultInfo(SQInfo *pQInfo) { } sem_wait(&pQInfo->dataReady); - dTrace("QInfo:%p retrieve result info, rowsize:%d, rows:%d, code:%d", pQInfo, pQuery->rowSize, pQuery->rec.size, + dTrace("QInfo:%p retrieve result info, rowsize:%d, rows:%d, code:%d", pQInfo, pQuery->rowSize, pQuery->rec.rows, pQInfo->code); return pQInfo->code; @@ -6167,12 +6167,12 @@ int32_t qDumpRetrieveResult(SQInfo *pQInfo, SRetrieveTableRsp** pRsp, int32_t* c } SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - size_t size = getResultSize(pQInfo, &pQuery->rec.size); + size_t size = getResultSize(pQInfo, &pQuery->rec.rows); *contLen = size + sizeof(SRetrieveTableRsp); // todo handle failed to allocate memory *pRsp = (SRetrieveTableRsp *)rpcMallocCont(*contLen); - (*pRsp)->numOfRows = htonl(pQuery->rec.size); + (*pRsp)->numOfRows = htonl(pQuery->rec.rows); int32_t code = pQInfo->code; if (code == TSDB_CODE_SUCCESS) { @@ -6183,7 +6183,7 @@ int32_t qDumpRetrieveResult(SQInfo *pQInfo, SRetrieveTableRsp** pRsp, int32_t* c (*pRsp)->useconds = 0; } - if (pQuery->rec.size > 0 && code == TSDB_CODE_SUCCESS) { + if (pQuery->rec.rows > 0 && code == TSDB_CODE_SUCCESS) { code = doDumpQueryResult(pQInfo, (*pRsp)->data); } else { setQueryStatus(pQuery, QUERY_OVER); diff --git a/src/query/src/sql.c b/src/query/src/sql.c index f1dce997fef02772ad6b0effa8084e8c26e9ef4e..d52e58395a13e33dc6cfbfc6565d1d2d2e235781 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -2471,13 +2471,13 @@ static void yy_reduce( yymsp[-1].minor.yy380 = yylhsminor.yy380; break; case 152: /* sortorder ::= ASC */ -{yymsp[0].minor.yy250 = TSQL_SO_ASC; } +{yymsp[0].minor.yy250 = TSDB_ORDER_ASC; } break; case 153: /* sortorder ::= DESC */ -{yymsp[0].minor.yy250 = TSQL_SO_DESC;} +{yymsp[0].minor.yy250 = TSDB_ORDER_DESC;} break; case 154: /* sortorder ::= */ -{yymsp[1].minor.yy250 = TSQL_SO_ASC;} +{yymsp[1].minor.yy250 = TSDB_ORDER_ASC;} break; case 157: /* grouplist ::= grouplist COMMA item */ { diff --git a/src/query/tests/astTest.cpp b/src/query/tests/astTest.cpp index 3b1f36e90f9fc4001fcf11d5dbcf105985787205..0f41ebea2667c1715d22e98a5968bc4ce6ad3792 100644 --- a/src/query/tests/astTest.cpp +++ b/src/query/tests/astTest.cpp @@ -255,7 +255,7 @@ static void testQueryStr(SSchema *schema, int32_t numOfCols, char *sql, tSkipLis printf("expr is: %s\n", str); SArray *result = NULL; - // tSQLBinaryExprTraverse(pExpr, pSkipList, result, tSkipListNodeFilterCallback, &result); + // tExprTreeTraverse(pExpr, pSkipList, result, tSkipListNodeFilterCallback, &result); // printf("the result is:%lld\n", result.num); // // bool findResult = false; @@ -533,7 +533,7 @@ tExprNode* createExpr2() { auto *p2 = (tExprNode*) calloc(1, sizeof(tExprNode)); p2->nodeType = TSQL_NODE_EXPR; - p2->_node.optr = TSDB_RELATION_LARGE_EQUAL; + p2->_node.optr = TSDB_RELATION_GREATER_EQUAL; p2->_node.pLeft = pLeft1; p2->_node.pRight = pRight1; p2->_node.hasPK = false; @@ -556,7 +556,8 @@ void exprSerializeTest1() { ASSERT_TRUE(size > 0); char* b = tbufGetData(&buf, false); - tExprNode* p2 = exprTreeFromBinary(b, size); + tExprNode* p2 = NULL; + exprTreeFromBinary(b, size, &p2); ASSERT_EQ(p1->nodeType, p2->nodeType); ASSERT_EQ(p2->_node.optr, p1->_node.optr); @@ -592,7 +593,8 @@ void exprSerializeTest2() { ASSERT_TRUE(size > 0); char* b = tbufGetData(&buf, false); - tExprNode* p2 = exprTreeFromBinary(b, size); + tExprNode* p2 = NULL; + exprTreeFromBinary(b, size, &p2); ASSERT_EQ(p1->nodeType, p2->nodeType); ASSERT_EQ(p2->_node.optr, p1->_node.optr); @@ -617,7 +619,7 @@ void exprSerializeTest2() { ASSERT_EQ(c1Right->nodeType, c2Right->nodeType); ASSERT_EQ(c2Right->nodeType, TSQL_NODE_EXPR); - ASSERT_EQ(c2Right->_node.optr, TSDB_RELATION_LARGE_EQUAL); + ASSERT_EQ(c2Right->_node.optr, TSDB_RELATION_GREATER_EQUAL); ASSERT_EQ(c2Right->_node.pRight->pVal->dKey, 91.99); ASSERT_EQ(p2->_node.hasPK, true); diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp index ff58e2a97446da04f96cc23fe224632fa7f7ad49..f50e1a5a718b0a197c602acec7dc6249410e3e21 100644 --- a/src/query/tests/tsBufTest.cpp +++ b/src/query/tests/tsBufTest.cpp @@ -37,7 +37,7 @@ void simpleTest() { int64_t* list = createTsList(10, 10000000, 30); tsBufAppend(pTSBuf, 0, tag, (const char*)list, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, sizeof(int64_t) * num); EXPECT_EQ(pTSBuf->block.tag, tag); @@ -65,7 +65,7 @@ void largeTSTest() { EXPECT_EQ(pTSBuf->tsData.len, 0); EXPECT_EQ(pTSBuf->block.tag, tag); EXPECT_EQ(pTSBuf->numOfVnodes, 1); - EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); tsBufFlush(pTSBuf); EXPECT_EQ(pTSBuf->tsData.len, 0); @@ -91,7 +91,7 @@ void multiTagsTest() { start += step * num; } - EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); @@ -127,7 +127,7 @@ void multiVnodeTagsTest() { EXPECT_EQ(pTSBuf->numOfVnodes, j + 1); } - EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); @@ -167,7 +167,7 @@ void loadDataTest() { EXPECT_EQ(pTSBuf->numOfVnodes, j + 1); } - EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); @@ -252,7 +252,7 @@ void TSTraverse() { int64_t s = taosGetTimestampUs(); printf("start:%" PRIu64 "\n", s); - pTSBuf->cur.order = TSQL_SO_DESC; + pTSBuf->cur.order = TSDB_ORDER_DESC; // complete reverse traverse int32_t x = 0; @@ -263,7 +263,7 @@ void TSTraverse() { // specify the data block with vnode and tags value tsBufResetPos(pTSBuf); - pTSBuf->cur.order = TSQL_SO_DESC; + pTSBuf->cur.order = TSDB_ORDER_DESC; int32_t startVnode = 1; int32_t startTag = 2; @@ -297,7 +297,7 @@ void TSTraverse() { ///////////////////////////////////////////////////////////////////////////////// // traverse - pTSBuf->cur.order = TSQL_SO_ASC; + pTSBuf->cur.order = TSDB_ORDER_ASC; tsBufResetPos(pTSBuf); // complete forwards traverse @@ -308,7 +308,7 @@ void TSTraverse() { // specify the data block with vnode and tags value tsBufResetPos(pTSBuf); - pTSBuf->cur.order = TSQL_SO_ASC; + pTSBuf->cur.order = TSDB_ORDER_ASC; startVnode = 1; startTag = 2; diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h index 6b70780d10482610ef64b1c50640756db4204bd6..28ebb1823582cad97af5ff2ed5c406d8edeb9e16 100644 --- a/src/util/inc/tarray.h +++ b/src/util/inc/tarray.h @@ -100,6 +100,12 @@ void taosArrayRemove(SArray* pArray, size_t index); */ void taosArrayCopy(SArray* pDst, SArray* pSrc); +/** + * clone a new array + * @param pSrc + */ +SArray* taosArrayClone(SArray* pSrc); + /** * destroy array list * @param pArray diff --git a/src/util/inc/tcompare.h b/src/util/inc/tcompare.h new file mode 100644 index 0000000000000000000000000000000000000000..3ac0d3d1703c2da27b4a277bd877ef9e32a9e402 --- /dev/null +++ b/src/util/inc/tcompare.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TCOMPARE_H +#define TDENGINE_TCOMPARE_H + +#include "os.h" + +int32_t compareInt32Val(const void *pLeft, const void *pRight); + +int32_t compareInt64Val(const void *pLeft, const void *pRight); + +int32_t compareInt16Val(const void *pLeft, const void *pRight); + +int32_t compareInt8Val(const void *pLeft, const void *pRight); + +int32_t compareIntDoubleVal(const void *pLeft, const void *pRight); + +int32_t compareDoubleIntVal(const void *pLeft, const void *pRight); + +int32_t compareDoubleVal(const void *pLeft, const void *pRight); + +int32_t compareStrVal(const void *pLeft, const void *pRight); + +int32_t compareWStrVal(const void *pLeft, const void *pRight); + +__compar_fn_t getKeyComparFunc(int32_t keyType); + +__compar_fn_t getComparFunc(int32_t type, int32_t filterDataType); + +#endif // TDENGINE_TCOMPARE_H diff --git a/src/util/inc/tskiplist.h b/src/util/inc/tskiplist.h index 18404f89c27d2c122e813edae674bb3869c451b7..dc67d6dad51f1b1c0835110d0e639a082fec4711 100644 --- a/src/util/inc/tskiplist.h +++ b/src/util/inc/tskiplist.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TBASE_TSKIPLIST_H -#define TBASE_TSKIPLIST_H +#ifndef TDENGINE_TSKIPLIST_H +#define TDENGINE_TSKIPLIST_H #ifdef __cplusplus extern "C" { @@ -24,10 +24,6 @@ extern "C" { #include "taosdef.h" #include "tarray.h" -/* - * key of each node - * todo move to as the global structure in all search codes... - */ #define MAX_SKIP_LIST_LEVEL 15 #define SKIP_LIST_RECORD_PERFORMANCE 0 @@ -35,11 +31,11 @@ typedef char *SSkipListKey; typedef char *(*__sl_key_fn_t)(const void *); /** + * the skip list node is located in a consecutive memory area, * the format of skip list node is as follows: * +------------+-----------------------+------------------------+-----+------+ * | node level | forward pointer array | backward pointer array | key | data | * +------------+-----------------------+------------------------+-----+------+ - * the skiplist node is located in a consecutive memory area, key will not be copy to skip list */ typedef struct SSkipListNode { uint8_t level; @@ -136,7 +132,8 @@ typedef struct SSkipList { typedef struct SSkipListIterator { SSkipList * pSkipList; SSkipListNode *cur; - int64_t num; + int32_t step; // the number of nodes that have been checked already + int32_t order; // order of the iterator } SSkipListIterator; /** @@ -205,6 +202,15 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel); */ SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList); +/** + * create skip list iterator from the given node and specified the order + * @param pSkipList + * @param pNode start position, instead of the first node in skip list + * @param order traverse order of the iterator + * @return + */ +SSkipListIterator *tSkipListCreateIterFromVal(SSkipList* pSkipList, const char* val, int32_t type, int32_t order); + /** * forward the skip list iterator * @param iter @@ -245,4 +251,4 @@ void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode); } #endif -#endif // TBASE_TSKIPLIST_H +#endif // TDENGINE_TSKIPLIST_H diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h index b80aad1cebab3782ad7e2c4751275d854ef5f92c..c4c802a68881107da8a09044ea4eb4aa206c9340 100644 --- a/src/util/inc/tutil.h +++ b/src/util/inc/tutil.h @@ -121,10 +121,6 @@ int64_t strnatoi(char *num, int32_t len); char* strreplace(const char* str, const char* pattern, const char* rep); -#define POW2(x) ((x) * (x)) - -int32_t strdequote(char *src); - char *paGetToken(char *src, char **token, int32_t *tokenLen); void taosMsleep(int32_t mseconds); @@ -133,8 +129,6 @@ int32_t taosByteArrayToHexStr(char bytes[], int32_t len, char hexstr[]); int32_t taosHexStrToByteArray(char hexstr[], char bytes[]); -int64_t str2int64(char *str); - int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath); /** diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index eb1d2133174b59a0ba93830a51986cbc78dc4c4b..d97b220a40a83450179ebe8d9c10597b83843bc0 100755 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -160,6 +160,21 @@ void taosArrayCopy(SArray* pDst, SArray* pSrc) { pDst->size = pSrc->size; } +SArray* taosArrayClone(SArray* pSrc) { + assert(pSrc != NULL); + + if (pSrc->size == 0) { // empty array list + return taosArrayInit(8, pSrc->elemSize); + } + + SArray* dst = taosArrayInit(pSrc->size, pSrc->elemSize); + + memcpy(dst->pData, pSrc->pData, pSrc->elemSize * pSrc->size); + dst->size = pSrc->size; + return dst; +} + + void taosArrayDestroy(SArray* pArray) { if (pArray == NULL) { return; diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c new file mode 100644 index 0000000000000000000000000000000000000000..401db9279a23e9462d472ce30ea11c701704677e --- /dev/null +++ b/src/util/src/tcompare.c @@ -0,0 +1,192 @@ +#include "taosdef.h" +#include "tcompare.h" + +int32_t compareInt32Val(const void *pLeft, const void *pRight) { + int32_t ret = GET_INT32_VAL(pLeft) - GET_INT32_VAL(pRight); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } +} + +int32_t compareInt64Val(const void *pLeft, const void *pRight) { + int32_t ret = GET_INT64_VAL(pLeft) - GET_INT64_VAL(pRight); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } +} + +int32_t compareInt16Val(const void *pLeft, const void *pRight) { + int32_t ret = GET_INT16_VAL(pLeft) - GET_INT16_VAL(pRight); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } +} + +int32_t compareInt8Val(const void *pLeft, const void *pRight) { + int32_t ret = GET_INT8_VAL(pLeft) - GET_INT8_VAL(pRight); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } +} + +int32_t compareIntDoubleVal(const void *pLeft, const void *pRight) { + // int64_t lhs = ((SSkipListKey *)pLeft)->i64Key; + // double rhs = ((SSkipListKey *)pRight)->dKey; + // if (fabs(lhs - rhs) < FLT_EPSILON) { + // return 0; + // } else { + // return (lhs > rhs) ? 1 : -1; + // } + return 0; +} + +int32_t compareDoubleIntVal(const void *pLeft, const void *pRight) { + // double lhs = ((SSkipListKey *)pLeft)->dKey; + // int64_t rhs = ((SSkipListKey *)pRight)->i64Key; + // if (fabs(lhs - rhs) < FLT_EPSILON) { + // return 0; + // } else { + // return (lhs > rhs) ? 1 : -1; + // } + return 0; +} + +int32_t compareDoubleVal(const void *pLeft, const void *pRight) { + double ret = GET_DOUBLE_VAL(pLeft) - GET_DOUBLE_VAL(pRight); + if (fabs(ret) < FLT_EPSILON) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } +} + +int32_t compareStrVal(const void *pLeft, const void *pRight) { + int32_t ret = strcmp(pLeft, pRight); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } +} + +int32_t compareWStrVal(const void *pLeft, const void *pRight) { + // SSkipListKey *pL = (SSkipListKey *)pLeft; + // SSkipListKey *pR = (SSkipListKey *)pRight; + // + // if (pL->nLen == 0 && pR->nLen == 0) { + // return 0; + // } + // + // // handle only one-side bound compare situation, there is only lower bound or only upper bound + // if (pL->nLen == -1) { + // return 1; // no lower bound, lower bound is minimum, always return -1; + // } else if (pR->nLen == -1) { + // return -1; // no upper bound, upper bound is maximum situation, always return 1; + // } + // + // int32_t ret = wcscmp(((SSkipListKey *)pLeft)->wpz, ((SSkipListKey *)pRight)->wpz); + // + // if (ret == 0) { + // return 0; + // } else { + // return ret > 0 ? 1 : -1; + // } + return 0; +} + +__compar_fn_t getComparFunc(int32_t type, int32_t filterDataType) { + __compar_fn_t comparFn = NULL; + + switch (type) { + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: { + if (filterDataType == TSDB_DATA_TYPE_BIGINT) { + comparFn = compareInt64Val; + break; + } + } + case TSDB_DATA_TYPE_BOOL: { + if (filterDataType >= TSDB_DATA_TYPE_BOOL && filterDataType <= TSDB_DATA_TYPE_BIGINT) { + comparFn = compareInt32Val; + } else if (filterDataType >= TSDB_DATA_TYPE_FLOAT && filterDataType <= TSDB_DATA_TYPE_DOUBLE) { + comparFn = compareIntDoubleVal; + } + break; + } + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: { +// if (filterDataType >= TSDB_DATA_TYPE_BOOL && filterDataType <= TSDB_DATA_TYPE_BIGINT) { +// comparFn = compareDoubleIntVal; +// } else if (filterDataType >= TSDB_DATA_TYPE_FLOAT && filterDataType <= TSDB_DATA_TYPE_DOUBLE) { +// comparFn = compareDoubleVal; +// } + if (filterDataType == TSDB_DATA_TYPE_DOUBLE) { + comparFn = compareDoubleVal; + } + break; + } + case TSDB_DATA_TYPE_BINARY: + comparFn = compareStrVal; + break; + case TSDB_DATA_TYPE_NCHAR: + comparFn = compareWStrVal; + break; + default: + comparFn = compareInt32Val; + break; + } + + return comparFn; +} + +__compar_fn_t getKeyComparFunc(int32_t keyType) { + __compar_fn_t comparFn = NULL; + + switch (keyType) { + case TSDB_DATA_TYPE_TINYINT: + comparFn = compareInt8Val; + break; + case TSDB_DATA_TYPE_SMALLINT: + comparFn = compareInt16Val; + break; + case TSDB_DATA_TYPE_INT: + comparFn = compareInt32Val; + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + comparFn = compareInt64Val; + break; + case TSDB_DATA_TYPE_BOOL: + comparFn = compareInt32Val; + break; + + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + comparFn = compareDoubleVal; + break; + + case TSDB_DATA_TYPE_BINARY: + comparFn = compareStrVal; + break; + + case TSDB_DATA_TYPE_NCHAR: + comparFn = compareWStrVal; + break; + + default: + comparFn = compareInt32Val; + break; + } + + return comparFn; +} diff --git a/src/util/src/tglobalcfg.c b/src/util/src/tglobalcfg.c index 88a3be0e02750a780b2c2b65fca152444b3c7578..8ce4530dbf997b6673d4e8cdd903428ea0a07ddc 100644 --- a/src/util/src/tglobalcfg.c +++ b/src/util/src/tglobalcfg.c @@ -14,13 +14,12 @@ */ #include "os.h" -#include +#include "taosdef.h" +#include "taoserror.h" #include "tglobalcfg.h" #include "tkey.h" #include "tlog.h" -#include "taosdef.h" -#include "taoserror.h" #include "tsocket.h" #include "tsystem.h" #include "tutil.h" @@ -840,9 +839,8 @@ void tsReadGlobalLogConfig() { FILE * fp; char * line, *option, *value; - size_t len; int olen, vlen; - char fileName[128]; + char fileName[PATH_MAX] = {0}; mdebugFlag = 135; sdbDebugFlag = 135; @@ -858,22 +856,26 @@ void tsReadGlobalLogConfig() { wordfree(&full_path); tsReadLogOption("logDir", logDir); + sprintf(fileName, "%s/taos.cfg", configDir); fp = fopen(fileName, "r"); if (fp == NULL) { - printf("\noption file:%s not found, all options are set to system default\n", fileName); + printf("\nconfig file:%s not found, all variables are set to default\n", fileName); return; } - - line = NULL; + + size_t len = 1024; + line = calloc(1, len); + while (!feof(fp)) { - tfree(line); - line = option = value = NULL; - len = olen = vlen = 0; + memset(line, 0, len); + + option = value = NULL; + olen = vlen = 0; getline(&line, &len, fp); - if (line == NULL) break; - + line[len - 1] = 0; + paGetToken(line, &option, &olen); if (olen == 0) continue; option[olen] = 0; @@ -892,25 +894,26 @@ void tsReadGlobalLogConfig() { bool tsReadGlobalConfig() { tsInitGlobalConfig(); - FILE * fp; char * line, *option, *value, *value1; - size_t len; int olen, vlen, vlen1; - char fileName[128]; + char fileName[PATH_MAX] = {0}; sprintf(fileName, "%s/taos.cfg", configDir); - fp = fopen(fileName, "r"); - if (fp == NULL) { - } else { - line = NULL; + FILE* fp = fopen(fileName, "r"); + + size_t len = 1024; + line = calloc(1, len); + + if (fp != NULL) { while (!feof(fp)) { - tfree(line); - line = option = value = NULL; - len = olen = vlen = 0; + memset(line, 0, len); - getline(&line, &len, fp); - if (line == NULL) break; + option = value = NULL; + olen = vlen = 0; + getline(&line, &len, fp); + line[len - 1] = 0; + paGetToken(line, &option, &olen); if (olen == 0) continue; option[olen] = 0; @@ -922,14 +925,15 @@ bool tsReadGlobalConfig() { // For dataDir, the format is: // dataDir /mnt/disk1 0 paGetToken(value + vlen + 1, &value1, &vlen1); - + tsReadConfigOption(option, value); } - tfree(line); fclose(fp); } + tfree(line); + if (tsReadStorageConfig) { tsReadStorageConfig(); } @@ -983,6 +987,7 @@ bool tsReadGlobalConfig() { strcpy(tsLocalIp, tsPrivateIp); } + // todo refactor tsVersion = 0; for (int i = 0; i < 10; i++) { if (version[i] >= '0' && version[i] <= '9') { @@ -991,6 +996,7 @@ bool tsReadGlobalConfig() { break; } } + tsVersion = 10 * tsVersion; return true; diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 0ef99b724488ad32958ce5cbcc86b3df90dc9336..138c75c1977b3a40a4fc0291ff4d93f4cf21d4c5 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -17,6 +17,7 @@ #include "tlog.h" #include "tskiplist.h" #include "tutil.h" +#include "tcompare.h" __attribute__ ((unused)) static FORCE_INLINE void recordNodeEachLevel(SSkipList *pSkipList, int32_t level) { // record link count in each level #if SKIP_LIST_RECORD_PERFORMANCE @@ -72,196 +73,54 @@ memset(pNode, 0, SL_NODE_HEADER_SIZE(_l));\ static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode); static SSkipListNode* tSkipListDoAppend(SSkipList *pSkipList, SSkipListNode *pNode); - -int32_t compareInt32Val(const void *pLeft, const void *pRight) { - int32_t ret = GET_INT32_VAL(pLeft) - GET_INT32_VAL(pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } -} - -int32_t compareInt64Val(const void *pLeft, const void *pRight) { - int32_t ret = GET_INT64_VAL(pLeft) - GET_INT64_VAL(pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } -} - -int32_t compareInt16Val(const void *pLeft, const void *pRight) { - int32_t ret = GET_INT16_VAL(pLeft) - GET_INT16_VAL(pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } -} - -int32_t compareInt8Val(const void *pLeft, const void *pRight) { - int32_t ret = GET_INT8_VAL(pLeft) - GET_INT8_VAL(pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } -} - -int32_t compareIntDoubleVal(const void *pLeft, const void *pRight) { - // int64_t lhs = ((SSkipListKey *)pLeft)->i64Key; - // double rhs = ((SSkipListKey *)pRight)->dKey; - // if (fabs(lhs - rhs) < FLT_EPSILON) { - // return 0; - // } else { - // return (lhs > rhs) ? 1 : -1; - // } - return 0; -} - -int32_t compareDoubleIntVal(const void *pLeft, const void *pRight) { - // double lhs = ((SSkipListKey *)pLeft)->dKey; - // int64_t rhs = ((SSkipListKey *)pRight)->i64Key; - // if (fabs(lhs - rhs) < FLT_EPSILON) { - // return 0; - // } else { - // return (lhs > rhs) ? 1 : -1; - // } - return 0; -} - -int32_t compareDoubleVal(const void *pLeft, const void *pRight) { - double ret = GET_DOUBLE_VAL(pLeft) - GET_DOUBLE_VAL(pRight); - if (fabs(ret) < FLT_EPSILON) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } -} - -int32_t compareStrVal(const void *pLeft, const void *pRight) { - int32_t ret = strcmp(pLeft, pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } -} - -int32_t compareWStrVal(const void *pLeft, const void *pRight) { - // SSkipListKey *pL = (SSkipListKey *)pLeft; - // SSkipListKey *pR = (SSkipListKey *)pRight; - // - // if (pL->nLen == 0 && pR->nLen == 0) { - // return 0; - // } - // - // // handle only one-side bound compare situation, there is only lower bound or only upper bound - // if (pL->nLen == -1) { - // return 1; // no lower bound, lower bound is minimum, always return -1; - // } else if (pR->nLen == -1) { - // return -1; // no upper bound, upper bound is maximum situation, always return 1; - // } - // - // int32_t ret = wcscmp(((SSkipListKey *)pLeft)->wpz, ((SSkipListKey *)pRight)->wpz); - // - // if (ret == 0) { - // return 0; - // } else { - // return ret > 0 ? 1 : -1; - // } - return 0; -} - -static __compar_fn_t getKeyFilterComparator(SSkipList *pSkipList, int32_t filterDataType) { - __compar_fn_t comparFn = NULL; - - switch (pSkipList->keyInfo.type) { - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: { - if (filterDataType == TSDB_DATA_TYPE_BIGINT) { - comparFn = compareInt64Val; - break; - } - } - case TSDB_DATA_TYPE_BOOL: { - if (filterDataType >= TSDB_DATA_TYPE_BOOL && filterDataType <= TSDB_DATA_TYPE_BIGINT) { - comparFn = compareInt32Val; - } else if (filterDataType >= TSDB_DATA_TYPE_FLOAT && filterDataType <= TSDB_DATA_TYPE_DOUBLE) { - comparFn = compareIntDoubleVal; - } - break; - } - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: { +static SSkipListIterator* doCreateSkipListIterator(SSkipList *pSkipList, int32_t order); + +//static __compar_fn_t getComparFunc(SSkipList *pSkipList, int32_t filterDataType) { +// __compar_fn_t comparFn = NULL; +// +// switch (pSkipList->keyInfo.type) { +// case TSDB_DATA_TYPE_TINYINT: +// case TSDB_DATA_TYPE_SMALLINT: +// case TSDB_DATA_TYPE_INT: +// case TSDB_DATA_TYPE_BIGINT: { +// if (filterDataType == TSDB_DATA_TYPE_BIGINT) { +// comparFn = compareInt64Val; +// break; +// } +// } +// case TSDB_DATA_TYPE_BOOL: { // if (filterDataType >= TSDB_DATA_TYPE_BOOL && filterDataType <= TSDB_DATA_TYPE_BIGINT) { -// comparFn = compareDoubleIntVal; +// comparFn = compareInt32Val; // } else if (filterDataType >= TSDB_DATA_TYPE_FLOAT && filterDataType <= TSDB_DATA_TYPE_DOUBLE) { +// comparFn = compareIntDoubleVal; +// } +// break; +// } +// case TSDB_DATA_TYPE_FLOAT: +// case TSDB_DATA_TYPE_DOUBLE: { +//// if (filterDataType >= TSDB_DATA_TYPE_BOOL && filterDataType <= TSDB_DATA_TYPE_BIGINT) { +//// comparFn = compareDoubleIntVal; +//// } else if (filterDataType >= TSDB_DATA_TYPE_FLOAT && filterDataType <= TSDB_DATA_TYPE_DOUBLE) { +//// comparFn = compareDoubleVal; +//// } +// if (filterDataType == TSDB_DATA_TYPE_DOUBLE) { // comparFn = compareDoubleVal; // } - if (filterDataType == TSDB_DATA_TYPE_DOUBLE) { - comparFn = compareDoubleVal; - } - break; - } - case TSDB_DATA_TYPE_BINARY: - comparFn = compareStrVal; - break; - case TSDB_DATA_TYPE_NCHAR: - comparFn = compareWStrVal; - break; - default: - comparFn = compareInt32Val; - break; - } - - return comparFn; -} - -static __compar_fn_t getKeyComparator(int32_t keyType) { - __compar_fn_t comparFn = NULL; - - switch (keyType) { - case TSDB_DATA_TYPE_TINYINT: - comparFn = compareInt8Val; - break; - case TSDB_DATA_TYPE_SMALLINT: - comparFn = compareInt16Val; - break; - case TSDB_DATA_TYPE_INT: - comparFn = compareInt32Val; - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: - comparFn = compareInt64Val; - break; - case TSDB_DATA_TYPE_BOOL: - comparFn = compareInt32Val; - break; - - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: - comparFn = compareDoubleVal; - break; - - case TSDB_DATA_TYPE_BINARY: - comparFn = compareStrVal; - break; - - case TSDB_DATA_TYPE_NCHAR: - comparFn = compareWStrVal; - break; - - default: - comparFn = compareInt32Val; - break; - } - - return comparFn; -} +// break; +// } +// case TSDB_DATA_TYPE_BINARY: +// comparFn = compareStrVal; +// break; +// case TSDB_DATA_TYPE_NCHAR: +// comparFn = compareWStrVal; +// break; +// default: +// comparFn = compareInt32Val; +// break; +// } +// +// return comparFn; +//} static bool initForwardBackwardPtr(SSkipList* pSkipList) { uint32_t maxLevel = pSkipList->maxLevel; @@ -298,7 +157,7 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint8_t keyLen, ui pSkipList->keyInfo = (SSkipListKeyInfo){.type = keyType, .len = keyLen, .dupKey = dupKey, .freeNode = freeNode}; pSkipList->keyFn = fn; - pSkipList->comparFn = getKeyComparator(keyType); + pSkipList->comparFn = getKeyComparFunc(keyType); pSkipList->maxLevel = maxLevel; pSkipList->level = 1; @@ -406,22 +265,18 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) { SSkipListNode *px = pSkipList->pHead; SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; - bool identical = false; + int32_t ret = -1; for (int32_t i = pSkipList->level - 1; i >= 0; --i) { SSkipListNode *p = SL_GET_FORWARD_POINTER(px, i); while (p != pSkipList->pTail) { char *key = SL_GET_NODE_KEY(pSkipList, p); // if the forward element is less than the specified key, forward one step - int32_t ret = pSkipList->comparFn(key, newDatakey); + ret = pSkipList->comparFn(key, newDatakey); if (ret < 0) { px = p; p = SL_GET_FORWARD_POINTER(px, i); } else { - if (identical == false) { - identical = (ret == 0); - } - break; } } @@ -430,7 +285,7 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) { } // if the skip list does not allowed identical key inserted, the new data will be discarded. - if (pSkipList->keyInfo.dupKey == 0 && identical) { + if (pSkipList->keyInfo.dupKey == 0 && ret == 0) { if (pSkipList->lock) { pthread_rwlock_unlock(pSkipList->lock); } @@ -447,20 +302,13 @@ void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListN for (int32_t i = 0; i < pNode->level; ++i) { SSkipListNode *x = forward[i]; - -// if (x != pSkipList->pTail) { - SL_GET_BACKWARD_POINTER(pNode, i) = x; + SL_GET_BACKWARD_POINTER(pNode, i) = x; - SSkipListNode *next = SL_GET_FORWARD_POINTER(x, i); -// if (next) { - SL_GET_BACKWARD_POINTER(next, i) = pNode; -// } + SSkipListNode *next = SL_GET_FORWARD_POINTER(x, i); + SL_GET_BACKWARD_POINTER(next, i) = pNode; - SL_GET_FORWARD_POINTER(pNode, i) = next; - SL_GET_FORWARD_POINTER(x, i) = pNode; -// } else { -// SL_GET_FORWARD_POINTER(pSkipList->pHead, i) = pNode; -// } + SL_GET_FORWARD_POINTER(pNode, i) = next; + SL_GET_FORWARD_POINTER(x, i) = pNode; } atomic_add_fetch_32(&pSkipList->size, 1); @@ -494,11 +342,9 @@ SSkipListNode* tSkipListDoAppend(SSkipList *pSkipList, SSkipListNode *pNode) { SArray* tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey, int16_t keyType) { int32_t sLevel = pSkipList->level - 1; - int32_t ret = -1; - + // result list SArray* sa = taosArrayInit(1, POINTER_BYTES); - SSkipListNode *pNode = pSkipList->pHead; if (pSkipList->lock) { @@ -509,15 +355,16 @@ SArray* tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey, int16_t keyType) { pSkipList->state.queryCount++; #endif - __compar_fn_t filterComparFn = getKeyFilterComparator(pSkipList, keyType); - + __compar_fn_t filterComparFn = getComparFunc(pSkipList->keyInfo.type, keyType); + int32_t ret = -1; for (int32_t i = sLevel; i >= 0; --i) { - SSkipListNode *pNext = SL_GET_FORWARD_POINTER(pNode, i); - while (pNext != NULL) { - char *key = SL_GET_NODE_KEY(pSkipList, pNext); + SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, i); + while (p != pSkipList->pTail) { + char *key = SL_GET_NODE_KEY(pSkipList, p); + if ((ret = filterComparFn(key, pKey)) < 0) { - pNode = pNext; - pNext = SL_GET_FORWARD_POINTER(pNext, i); + pNode = p; + p = SL_GET_FORWARD_POINTER(p, i); } else { break; } @@ -525,10 +372,6 @@ SArray* tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey, int16_t keyType) { // find the qualified key if (ret == 0) { - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } - SSkipListNode* pResult = SL_GET_FORWARD_POINTER(pNode, i); taosArrayPush(sa, &pResult); @@ -559,25 +402,57 @@ SSkipListIterator* tSkipListCreateIter(SSkipList *pSkipList) { return NULL; } - SSkipListIterator* iter = calloc(1, sizeof(SSkipListIterator)); - - iter->pSkipList = pSkipList; - if (pSkipList->lock) { - pthread_rwlock_rdlock(pSkipList->lock); + return doCreateSkipListIterator(pSkipList, TSDB_ORDER_ASC); +} + +SSkipListIterator *tSkipListCreateIterFromVal(SSkipList* pSkipList, const char* val, int32_t type, int32_t order) { + if (pSkipList == NULL) { + return NULL; } - iter->cur = NULL; - iter->num = pSkipList->size; + assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } + if (val == NULL) { + return doCreateSkipListIterator(pSkipList, order); + } else { - return iter; + SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; + + int32_t ret = -1; + __compar_fn_t filterComparFn = getComparFunc(pSkipList->keyInfo.type, type); + SSkipListNode* pNode = pSkipList->pHead; + + for (int32_t i = pSkipList->level - 1; i >= 0; --i) { + SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, i); + while (p != pSkipList->pTail) { + char *key = SL_GET_NODE_KEY(pSkipList, p); + + if ((ret = filterComparFn(key, val)) < 0) { + pNode = p; + p = SL_GET_FORWARD_POINTER(p, i); + } else { + break; + } + } + + forward[i] = pNode; + } + + SSkipListIterator* iter = doCreateSkipListIterator(pSkipList, order); + + // set the initial position + if (order == TSDB_ORDER_ASC) { + iter->cur = forward[0]; // greater equals than the value + } else { + iter->cur = SL_GET_FORWARD_POINTER(forward[0], 0); + } + + return iter; + } } bool tSkipListIterNext(SSkipListIterator *iter) { - if (iter->num == 0 || iter->pSkipList == NULL) { + if (iter->pSkipList == NULL) { return false; } @@ -587,26 +462,35 @@ bool tSkipListIterNext(SSkipListIterator *iter) { pthread_rwlock_rdlock(pSkipList->lock); } - if (iter->cur == NULL) { - iter->cur = SL_GET_FORWARD_POINTER(pSkipList->pHead, 0); - } else { - iter->cur = SL_GET_FORWARD_POINTER(iter->cur, 0); + if (iter->order == TSDB_ORDER_ASC) { // ascending order iterate + if (iter->cur == NULL) { + iter->cur = SL_GET_FORWARD_POINTER(pSkipList->pHead, 0); + } else { + iter->cur = SL_GET_FORWARD_POINTER(iter->cur, 0); + } + } else { // descending order iterate + if (iter->cur == NULL) { + iter->cur = SL_GET_BACKWARD_POINTER(pSkipList->pTail, 0); + } else { + iter->cur = SL_GET_BACKWARD_POINTER(iter->cur, 0); + } } if (pSkipList->lock) { pthread_rwlock_unlock(pSkipList->lock); } - return iter->cur != pSkipList->pTail; + return (iter->order == TSDB_ORDER_ASC)? (iter->cur != pSkipList->pTail) : (iter->cur != pSkipList->pHead); } SSkipListNode *tSkipListIterGet(SSkipListIterator *iter) { - if (iter == NULL || iter->cur == iter->pSkipList->pTail) { + if (iter == NULL || iter->cur == iter->pSkipList->pTail || iter->cur == iter->pSkipList->pHead) { return NULL; } else { return iter->cur; } } + void* tSkipListDestroyIter(SSkipListIterator* iter) { if (iter == NULL) { return NULL; @@ -622,7 +506,7 @@ void* tSkipListDestroyIter(SSkipListIterator* iter) { // SSkipListNode *p = pStartNode; // int32_t numOfRes = 0; // -// __compar_fn_t filterComparFn = getKeyFilterComparator(pSkipList, pEndKey->nType); +// __compar_fn_t filterComparFn = getComparFunc(pSkipList, pEndKey->nType); // while (p != NULL) { // int32_t ret = filterComparFn(&p->key, pEndKey); // if (ret > 0) { @@ -663,7 +547,7 @@ void* tSkipListDestroyIter(SSkipListIterator* iter) { // return 0; // } // -// __compar_fn_t filterComparFn = getKeyFilterComparator(pSkipList, pKey->nType); +// __compar_fn_t filterComparFn = getComparFunc(pSkipList, pKey->nType); // // // backward check if previous nodes are with the same value. // SSkipListNode *pPrev = pNode->pBackward[0]; @@ -679,11 +563,11 @@ void* tSkipListDestroyIter(SSkipListIterator* iter) { // int32_t ret = -1; // // SSkipListNode *x = &pSkipList->pHead; -// __compar_fn_t filterComparFn = getKeyFilterComparator(pSkipList, pKey->nType); +// __compar_fn_t filterComparFn = getComparFunc(pSkipList, pKey->nType); // // pthread_rwlock_rdlock(&pSkipList->lock); // -// if (cond == TSDB_RELATION_LARGE_EQUAL || cond == TSDB_RELATION_LARGE) { +// if (cond == TSDB_RELATION_GREATER_EQUAL || cond == TSDB_RELATION_GREATER) { // for (int32_t i = sLevel; i >= 0; --i) { // while (x->pForward[i] != NULL && (ret = filterComparFn(&x->pForward[i]->key, pKey)) < 0) { // x = x->pForward[i]; @@ -691,7 +575,7 @@ void* tSkipListDestroyIter(SSkipListIterator* iter) { // } // // // backward check if previous nodes are with the same value. -// if (cond == TSDB_RELATION_LARGE_EQUAL && ret == 0) { +// if (cond == TSDB_RELATION_GREATER_EQUAL && ret == 0) { // SSkipListNode *pNode = x->pForward[0]; // while ((pNode->pBackward[0] != &pSkipList->pHead) && (filterComparFn(&pNode->pBackward[0]->key, pKey) == 0)) { // pNode = pNode->pBackward[0]; @@ -700,10 +584,10 @@ void* tSkipListDestroyIter(SSkipListIterator* iter) { // return pNode; // } // -// if (ret > 0 || cond == TSDB_RELATION_LARGE_EQUAL) { +// if (ret > 0 || cond == TSDB_RELATION_GREATER_EQUAL) { // pthread_rwlock_unlock(&pSkipList->lock); // return x->pForward[0]; -// } else { // cond == TSDB_RELATION_LARGE && ret == 0 +// } else { // cond == TSDB_RELATION_GREATER && ret == 0 // SSkipListNode *pn = x->pForward[0]; // while (pn != NULL && filterComparFn(&pn->key, pKey) == 0) { // pn = pn->pForward[0]; @@ -717,19 +601,9 @@ void* tSkipListDestroyIter(SSkipListIterator* iter) { // return NULL; //} // -// int32_t tSkipListRangeQuery(SSkipList *pSkipList, tSKipListQueryCond *pCond, SSkipListNode ***pRes) { -// pSkipList->state.queryCount++; -// SSkipListNode *pStart = tSkipListParQuery(pSkipList, &pCond->lowerBnd, pCond->lowerBndRelOptr); -// if (pStart == 0) { -// *pRes = NULL; -// return 0; -// } -// -// return tSkipListEndParQuery(pSkipList, pStart, &pCond->upperBnd, pCond->upperBndRelOptr, pRes); -//} // // static bool removeSupport(SSkipList *pSkipList, SSkipListNode **forward, SSkipListKey *pKey) { -// __compar_fn_t filterComparFn = getKeyFilterComparator(pSkipList, pKey->nType); +// __compar_fn_t filterComparFn = getComparFunc(pSkipList, pKey->nType); // // if (filterComparFn(&forward[0]->pForward[0]->key, pKey) == 0) { // SSkipListNode *p = forward[0]->pForward[0]; @@ -760,7 +634,7 @@ void* tSkipListDestroyIter(SSkipListIterator* iter) { // // bool tSkipListRemove(SSkipList *pSkipList, SSkipListKey *pKey) { // SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; -// __compar_fn_t filterComparFn = getKeyFilterComparator(pSkipList, pKey->nType); +// __compar_fn_t filterComparFn = getComparFunc(pSkipList, pKey->nType); // // pthread_rwlock_rdlock(&pSkipList->lock); // @@ -818,3 +692,12 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { p = SL_GET_FORWARD_POINTER(p, nlevel - 1); } } + +SSkipListIterator* doCreateSkipListIterator(SSkipList *pSkipList, int32_t order) { + SSkipListIterator* iter = calloc(1, sizeof(SSkipListIterator)); + + iter->pSkipList = pSkipList; + iter->order = order; + + return iter; +} diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index cadb2e706c12cf0871712b4cb9d58a42942e8a85..5f4c90327992ed523ede1491c8a2878ea5a6d9d2 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -166,7 +166,7 @@ char* strtolower(char *dst, const char *src) { char *paGetToken(char *string, char **token, int32_t *tokenLen) { char quote = 0; - + while (*string != 0) { if (*string == ' ' || *string == '\t') { ++string; @@ -336,7 +336,7 @@ int32_t taosByteArrayToHexStr(char bytes[], int32_t len, char hexstr[]) { char hexval[16] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; for (i = 0; i < len; i++) { - hexstr[i * 2] = hexval[((bytes[i] >> 4) & 0xF)]; + hexstr[i * 2] = hexval[((bytes[i] >> 4u) & 0xF)]; hexstr[(i * 2) + 1] = hexval[(bytes[i]) & 0x0F]; } diff --git a/src/util/tests/skiplistTest.cpp b/src/util/tests/skiplistTest.cpp index 9e690c3924062e67aea47974d0e2d08d78fa46b7..8bfa3a43c0476d0cc5a661c19df4422e663ce91c 100644 --- a/src/util/tests/skiplistTest.cpp +++ b/src/util/tests/skiplistTest.cpp @@ -104,6 +104,7 @@ void randKeyTest() { tSkipListDestroy(pSkipList); } + void stringKeySkiplistTest() { const int32_t max_key_size = 12; diff --git a/src/vnode/tsdb/inc/tsdb.h b/src/vnode/tsdb/inc/tsdb.h index b2673413ae9eb66247c9f4ab7c68baac694fcbfb..ff3d4666fca4bf9c5696bde544f8c8702375c99c 100644 --- a/src/vnode/tsdb/inc/tsdb.h +++ b/src/vnode/tsdb/inc/tsdb.h @@ -208,7 +208,7 @@ typedef struct SBlockInfo { typedef struct SDataBlockInfo { STimeWindow window; - int32_t size; + int32_t rows; int32_t numOfCols; int64_t uid; int32_t sid; @@ -335,7 +335,7 @@ SArray *tsdbGetTableList(tsdb_query_handle_t *pQueryHandle); * @param pTagCond. tag query condition * */ -SArray *tsdbQueryTableList(tsdb_repo_t* tsdb, int64_t uid, const wchar_t *pTagCond, size_t len); +int32_t tsdbQueryTags(tsdb_repo_t* tsdb, int64_t uid, const char *pTagCond, size_t len, SArray* list); /** * clean up the query handle diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index aa185f4bf2345ed0de74ed9aa46265c0a919eb08..62a43333aed5e9feb6bfa49944e6c607e70939a6 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -6,9 +6,10 @@ #include #include #include +#include #include #include -#include +#include #include // #include "taosdef.h" @@ -741,6 +742,7 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock) { STableId tableId = {.uid = pBlock->uid, .tid = pBlock->tid}; STable *pTable = tsdbIsValidTableToInsert(pRepo->tsdbMeta, tableId); if (pTable == NULL) { + dError("failed to get table for insert, uid:%" PRIu64 ", tid:%d", tableId.uid, tableId.tid); return TSDB_CODE_INVALID_TABLE_ID; } diff --git a/src/vnode/tsdb/src/tsdbMeta.c b/src/vnode/tsdb/src/tsdbMeta.c index 22f735713566edd8ae5740338855de5998a663b2..0eb6dde1d0b44c65a521f46e2dfdafdee069a16b 100644 --- a/src/vnode/tsdb/src/tsdbMeta.c +++ b/src/vnode/tsdb/src/tsdbMeta.c @@ -93,6 +93,11 @@ void tsdbFreeEncode(void *cont) { if (cont != NULL) free(cont); } +static char* getTagIndexKey(const void* pData) { + STable* table = *(STable**) pData; + return getTupleKey(table->tagVal); +} + int tsdbRestoreTable(void *pHandle, void *cont, int contLen) { STsdbMeta *pMeta = (STsdbMeta *)pHandle; @@ -101,8 +106,8 @@ int tsdbRestoreTable(void *pHandle, void *cont, int contLen) { if (pTable->type == TSDB_SUPER_TABLE) { pTable->pIndex = - tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1, 0, 0, getTupleKey); - } + tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1, 0, 0, getTagIndexKey); + } tsdbAddTableToMeta(pMeta, pTable, false); @@ -218,7 +223,7 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) { super->tagSchema = tdDupSchema(pCfg->tagSchema); super->tagVal = tdDataRowDup(pCfg->tagValues); super->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1, - 0, 0, getTupleKey); // Allow duplicate key, no lock + 0, 0, getTagIndexKey); // Allow duplicate key, no lock if (super->pIndex == NULL) { tdFreeSchema(super->schema); @@ -414,8 +419,6 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) { pNode->level = level; SSkipList* list = pSTable->pIndex; - - memcpy(SL_GET_NODE_KEY(list, pNode), dataRowTuple(pTable->tagVal), colBytes(s)); memcpy(SL_GET_NODE_DATA(pNode), &pTable, POINTER_BYTES); tSkipListPut(list, pNode); diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index a44174ae6613eccee7accaa9b14437c8b24e310a..30181bed1dfd0e533238958d4c9596e5c0fcfa22 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -19,13 +19,14 @@ #include "tutil.h" #include "../../../query/inc/qast.h" +#include "../../../query/inc/tlosertree.h" #include "../../../query/inc/tsqlfunction.h" #include "tsdb.h" #include "tsdbMain.h" #define EXTRA_BYTES 2 -#define PRIMARY_TSCOL_REQUIRED(c) (((SColumnInfoEx *)taosArrayGet(c, 0))->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) -#define QUERY_IS_ASC_QUERY(o) (o == TSQL_SO_ASC) +#define PRIMARY_TSCOL_REQUIRED(c) (((SColumnInfoEx*)taosArrayGet(c, 0))->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) +#define QUERY_IS_ASC_QUERY(o) (o == TSDB_ORDER_ASC) #define QH_GET_NUM_OF_COLS(handle) (taosArrayGetSize((handle)->pColumns)) enum { @@ -46,9 +47,9 @@ typedef struct SQueryFilePos { typedef struct SDataBlockLoadInfo { SFileGroup* fileGroup; - int32_t slot; - int32_t sid; - SArray *pLoadedCols; + int32_t slot; + int32_t sid; + SArray* pLoadedCols; } SDataBlockLoadInfo; typedef struct SLoadCompBlockInfo { @@ -57,52 +58,31 @@ typedef struct SLoadCompBlockInfo { int32_t fileListIndex; } SLoadCompBlockInfo; -typedef struct SQueryFilesInfo { - SArray *pFileInfo; - int32_t current; // the memory mapped header file, NOTE: only one header file can be mmap. - int32_t vnodeId; - - int32_t headerFd; // header file fd - int64_t headerFileSize; - int32_t dataFd; - int32_t lastFd; - - char headerFilePath[PATH_MAX]; // current opened header file name - char dataFilePath[PATH_MAX]; // current opened data file name - char lastFilePath[PATH_MAX]; // current opened last file path - char dbFilePathPrefix[PATH_MAX]; -} SQueryFilesInfo; - typedef struct STableCheckInfo { - STableId tableId; - TSKEY lastKey; - STable * pTableObj; - int64_t offsetInHeaderFile; -// int32_t numOfBlocks; - int32_t start; - bool checkFirstFileBlock; - - SCompIdx* compIndex; - SCompInfo *pCompInfo; - - SDataCols* pDataCols; - SFileGroup* pFileGroup; - - SFileGroupIter fileIter; + STableId tableId; + TSKEY lastKey; + STable* pTableObj; + int64_t offsetInHeaderFile; + int32_t start; + bool checkFirstFileBlock; + SCompInfo* pCompInfo; + int32_t numOfBlocks; // number of qualified data blocks not the original blocks + + SDataCols* pDataCols; SSkipListIterator* iter; } STableCheckInfo; typedef struct { - SCompBlock *compBlock; - SField * fields; + SCompBlock* compBlock; + SField* fields; } SCompBlockFields; -typedef struct STableDataBlockInfoEx { +typedef struct STableBlockInfo { SCompBlockFields pBlock; - STableCheckInfo* pMeterDataInfo; + STableCheckInfo* pTableCheckInfo; int32_t blockIndex; - int32_t groupIdx; /* number of group is less than the total number of meters */ -} STableDataBlockInfoEx; + int32_t groupIdx; /* number of group is less than the total number of tables */ +} STableBlockInfo; enum { SINGLE_TABLE_MODEL = 1, @@ -110,17 +90,13 @@ enum { }; typedef struct STsdbQueryHandle { - STsdbRepo* pTsdb; - int8_t model; // access model, single table model or multi-table model + STsdbRepo* pTsdb; + int8_t model; // access model, single table model or multi-table model SQueryFilePos cur; // current position SQueryFilePos start; // the start position, used for secondary/third iteration - int32_t unzipBufSize; - char *unzipBuffer; - char *secondaryUnzipBuffer; SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */ - SQueryFilesInfo vnodeFileInfo; int16_t numOfRowsPerPage; uint16_t flag; // denotes reversed scan of data or not @@ -129,24 +105,29 @@ typedef struct STsdbQueryHandle { int32_t blockBufferSize; SCompBlock* pBlock; int32_t numOfBlocks; - SField ** pFields; - SArray * pColumns; // column list, SColumnInfoEx array list + SField** pFields; + SArray* pColumns; // column list, SColumnInfoEx array list bool locateStart; int32_t realNumOfRows; bool loadDataAfterSeek; // load data after seek. SArray* pTableCheckInfo; int32_t activeIndex; - - int32_t tableIndex; - bool isFirstSlot; - void * qinfo; // query info handle, for debug purpose - - STableDataBlockInfoEx *pDataBlockInfoEx; + + bool checkFiles; // check file stage + int32_t tableIndex; + bool isFirstSlot; + void* qinfo; // query info handle, for debug purpose + + STableBlockInfo* pDataBlockInfo; + + SFileGroup* pFileGroup; + SFileGroupIter fileIter; + SCompIdx* compIndex; } STsdbQueryHandle; -int32_t doAllocateBuf(STsdbQueryHandle *pQueryHandle, int32_t rowsPerFileBlock) { +int32_t doAllocateBuf(STsdbQueryHandle* pQueryHandle, int32_t rowsPerFileBlock) { // record the maximum column width among columns of this meter/metric - SColumnInfoEx *pColumn = taosArrayGet(pQueryHandle->pColumns, 0); + SColumnInfoEx* pColumn = taosArrayGet(pQueryHandle->pColumns, 0); int32_t maxColWidth = pColumn->info.bytes; for (int32_t i = 1; i < QH_GET_NUM_OF_COLS(pQueryHandle); ++i) { @@ -156,89 +137,65 @@ int32_t doAllocateBuf(STsdbQueryHandle *pQueryHandle, int32_t rowsPerFileBlock) } } - // only one unzip buffer required, since we can unzip each column one by one - pQueryHandle->unzipBufSize = (size_t)(maxColWidth * rowsPerFileBlock + EXTRA_BYTES); // plus extra_bytes - pQueryHandle->unzipBuffer = (char *)calloc(1, pQueryHandle->unzipBufSize); - - pQueryHandle->secondaryUnzipBuffer = (char *)calloc(1, pQueryHandle->unzipBufSize); - - if (pQueryHandle->unzipBuffer == NULL || pQueryHandle->secondaryUnzipBuffer == NULL) { - goto _error_clean; - } - return TSDB_CODE_SUCCESS; - -_error_clean: - tfree(pQueryHandle->unzipBuffer); - tfree(pQueryHandle->secondaryUnzipBuffer); - - return TSDB_CODE_SERV_OUT_OF_MEMORY; } -static void initQueryFileInfoFD(SQueryFilesInfo *pVnodeFilesInfo) { - pVnodeFilesInfo->current = -1; - pVnodeFilesInfo->headerFileSize = -1; - - pVnodeFilesInfo->headerFd = FD_INITIALIZER; // set the initial value - pVnodeFilesInfo->dataFd = FD_INITIALIZER; - pVnodeFilesInfo->lastFd = FD_INITIALIZER; -} - -static void vnodeInitDataBlockLoadInfo(SDataBlockLoadInfo *pBlockLoadInfo) { +static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { pBlockLoadInfo->slot = -1; pBlockLoadInfo->sid = -1; pBlockLoadInfo->fileGroup = NULL; } -static void vnodeInitCompBlockLoadInfo(SLoadCompBlockInfo *pCompBlockLoadInfo) { +static void tsdbInitCompBlockLoadInfo(SLoadCompBlockInfo* pCompBlockLoadInfo) { pCompBlockLoadInfo->sid = -1; pCompBlockLoadInfo->fileId = -1; pCompBlockLoadInfo->fileListIndex = -1; } -tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond, SArray *idList, SArray *pColumnInfo) { +tsdb_query_handle_t* tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond* pCond, SArray* idList, SArray* pColumnInfo) { // todo 1. filter not exist table // todo 2. add the reference count for each table that is involved in query - STsdbQueryHandle *pQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); + STsdbQueryHandle* pQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); pQueryHandle->order = pCond->order; pQueryHandle->window = pCond->twindow; pQueryHandle->pTsdb = tsdb; + pQueryHandle->compIndex = calloc(10000, sizeof(SCompIdx)), pQueryHandle->loadDataAfterSeek = false; pQueryHandle->isFirstSlot = true; - + size_t size = taosArrayGetSize(idList); assert(size >= 1); pQueryHandle->pTableCheckInfo = taosArrayInit(size, sizeof(STableCheckInfo)); - for(int32_t i = 0; i < size; ++i) { - STableId id = *(STableId*) taosArrayGet(idList, i); - + for (int32_t i = 0; i < size; ++i) { + STableId id = *(STableId*)taosArrayGet(idList, i); + STableCheckInfo info = { - .lastKey = pQueryHandle->window.skey, - .tableId = id, - .pTableObj = tsdbGetTableByUid(tsdbGetMeta(tsdb), id.uid), //todo this may be failed - .compIndex = calloc(10000, sizeof(SCompIdx)), - .pCompInfo = calloc(1, 1024), + .lastKey = pQueryHandle->window.skey, + .tableId = id, + .pTableObj = tsdbGetTableByUid(tsdbGetMeta(tsdb), id.uid), // todo this may be failed + .pCompInfo = calloc(1, 1024), }; - + assert(info.pTableObj != NULL); taosArrayPush(pQueryHandle->pTableCheckInfo, &info); } - - pQueryHandle->model = (size > 1)? MULTI_TABLE_MODEL:SINGLE_TABLE_MODEL; - + + pQueryHandle->model = (size > 1) ? MULTI_TABLE_MODEL : SINGLE_TABLE_MODEL; + pQueryHandle->checkFiles = 1; + pQueryHandle->activeIndex = 0; - + // malloc buffer in order to load data from file int32_t numOfCols = taosArrayGetSize(pColumnInfo); size_t bufferCapacity = 4096; - + pQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoEx)); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoEx *pCol = taosArrayGet(pColumnInfo, i); + SColumnInfoEx* pCol = taosArrayGet(pColumnInfo, i); SColumnInfoEx pDest = {{0}, 0}; pDest.pData = calloc(1, EXTRA_BYTES + bufferCapacity * pCol->info.bytes); @@ -250,152 +207,224 @@ tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond return NULL; } - initQueryFileInfoFD(&pQueryHandle->vnodeFileInfo); - vnodeInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); - vnodeInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); + tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); + tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); return (tsdb_query_handle_t)pQueryHandle; } -static bool hasMoreDataInCacheForSingleModel(STsdbQueryHandle* pHandle) { +static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { assert(pHandle->activeIndex == 0 && taosArrayGetSize(pHandle->pTableCheckInfo) == 1); - + STableCheckInfo* pTableCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - - STable *pTable = pTableCheckInfo->pTableObj; + + STable* pTable = pTableCheckInfo->pTableObj; assert(pTable != NULL); - + // no data in cache, abort if (pTable->mem == NULL && pTable->imem == NULL) { return false; } - + // all data in mem are checked already. if (pTableCheckInfo->lastKey > pTable->mem->keyLast) { return false; } - + return true; } // todo dynamic get the daysperfile static int32_t getFileIdFromKey(TSKEY key) { - return (int32_t)(key / 10); // set the starting fileId + return (int32_t)(key / 10); // set the starting fileId } -static int32_t getFileCompInfo(STableCheckInfo* pCheckInfo, SFileGroup* fileGroup) { - // check open file failed +static int32_t binarySearchForBlockImpl(SCompBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order); + +static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlocks, int32_t type) { + // todo check open file failed + SFileGroup* fileGroup = pQueryHandle->pFileGroup; if (fileGroup->files[TSDB_FILE_TYPE_HEAD].fd == FD_INITIALIZER) { fileGroup->files[TSDB_FILE_TYPE_HEAD].fd = open(fileGroup->files[TSDB_FILE_TYPE_HEAD].fname, O_RDONLY); } - - tsdbLoadCompIdx(fileGroup, pCheckInfo->compIndex, 10000); // todo set dynamic max tables - SCompIdx* compIndex = &pCheckInfo->compIndex[pCheckInfo->tableId.tid]; - - if (compIndex->len == 0 || compIndex->numOfSuperBlocks == 0) { // no data block in this file, try next file - - } else { - tsdbLoadCompBlocks(fileGroup, compIndex, pCheckInfo->pCompInfo); + + // load all the comp offset value for all tables in this file + tsdbLoadCompIdx(fileGroup, pQueryHandle->compIndex, 10000); // todo set dynamic max tables + + *numOfBlocks = 0; + size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + + for (int32_t i = 0; i < numOfTables; ++i) { + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + + SCompIdx* compIndex = &pQueryHandle->compIndex[pCheckInfo->tableId.tid]; + if (compIndex->len == 0 || compIndex->numOfSuperBlocks == 0) { // no data block in this file, try next file + + } else { + tsdbLoadCompBlocks(fileGroup, compIndex, pCheckInfo->pCompInfo); + + int32_t index = 0; + // discard the unqualified data block based on the query time window + int32_t start = binarySearchForBlockImpl(pCheckInfo->pCompInfo->blocks, compIndex->numOfSuperBlocks, + pQueryHandle->order, pCheckInfo->lastKey); + + if (type == QUERY_RANGE_GREATER_EQUAL) { + if (pCheckInfo->lastKey <= pCheckInfo->pCompInfo->blocks[start].keyLast) { + // break; + } else { + index = -1; + } + } else { + if (pCheckInfo->lastKey >= pCheckInfo->pCompInfo->blocks[start].keyFirst) { + // break; + } else { + index = -1; + } + } + + // not found in data blocks in current file + if (index == -1) { + continue; + } + + // todo speedup the procedure of locating end block + int32_t e = start; + while (e < compIndex->numOfSuperBlocks && + (pCheckInfo->pCompInfo->blocks[e].keyFirst <= pQueryHandle->window.ekey)) { + e += 1; + } + + if (start > 0) { + memmove(pCheckInfo->pCompInfo->blocks, &pCheckInfo->pCompInfo->blocks[start], (e - start) * sizeof(SCompBlock)); + } + + pCheckInfo->numOfBlocks = (e - start); + (*numOfBlocks) += pCheckInfo->numOfBlocks; + } } - + return TSDB_CODE_SUCCESS; } -static int32_t binarySearchForBlockImpl(SCompBlock *pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) { +int32_t binarySearchForBlockImpl(SCompBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) { int32_t firstSlot = 0; int32_t lastSlot = numOfBlocks - 1; - + int32_t midSlot = firstSlot; - + while (1) { numOfBlocks = lastSlot - firstSlot + 1; midSlot = (firstSlot + (numOfBlocks >> 1)); - + if (numOfBlocks == 1) break; - + if (skey > pBlock[midSlot].keyLast) { if (numOfBlocks == 2) break; - if ((order == TSQL_SO_DESC) && (skey < pBlock[midSlot + 1].keyFirst)) break; + if ((order == TSDB_ORDER_DESC) && (skey < pBlock[midSlot + 1].keyFirst)) break; firstSlot = midSlot + 1; } else if (skey < pBlock[midSlot].keyFirst) { - if ((order == TSQL_SO_ASC) && (skey > pBlock[midSlot - 1].keyLast)) break; + if ((order == TSDB_ORDER_ASC) && (skey > pBlock[midSlot - 1].keyLast)) break; lastSlot = midSlot - 1; } else { break; // got the slot } } - + return midSlot; } -static SDataBlockInfo getTrueDataBlockInfo(STsdbQueryHandle* pHandle, STableCheckInfo* pCheckInfo) { - SCompBlock *pDiskBlock = &pCheckInfo->pCompInfo->blocks[pHandle->cur.slot]; - +static SDataBlockInfo getTrueDataBlockInfo(STableCheckInfo* pCheckInfo, SCompBlock* pBlock) { SDataBlockInfo info = { - .window = {.skey = pDiskBlock->keyFirst, .ekey = pDiskBlock->keyLast}, - .numOfCols = pDiskBlock->numOfCols, - .size = pDiskBlock->numOfPoints, - .sid = pCheckInfo->tableId.tid, - .uid = pCheckInfo->tableId.uid, + .window = {.skey = pBlock->keyFirst, .ekey = pBlock->keyLast}, + .numOfCols = pBlock->numOfCols, + .rows = pBlock->numOfPoints, + .sid = pCheckInfo->tableId.tid, + .uid = pCheckInfo->tableId.uid, }; - + return info; } -SArray *getDefaultLoadColumns(STsdbQueryHandle *pQueryHandle, bool loadTS); -static void filterDataInDataBlock(STsdbQueryHandle *pQueryHandle, SDataCols* pCols, SArray *sa); +SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS); +static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock, + SArray* sa); +static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); + +static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo) { + SCompData* data = calloc(1, sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols); -static bool doLoadDataFromFileBlock(STsdbQueryHandle *pQueryHandle) { - SQueryFilePos *cur = &pQueryHandle->cur; - - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); - SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; - - SCompData* data = calloc(1, sizeof(SCompData)+ sizeof(SCompCol)*pBlock->numOfCols); - data->numOfCols = pBlock->numOfCols; data->uid = pCheckInfo->pTableObj->tableId.uid; - - pCheckInfo->pDataCols = tdNewDataCols(1000, 2, 4096); - tdInitDataCols(pCheckInfo->pDataCols, pCheckInfo->pTableObj->schema); - - SFile* pFile = &pCheckInfo->pFileGroup->files[TSDB_FILE_TYPE_DATA]; + + bool blockLoaded = false; + SArray* sa = getDefaultLoadColumns(pQueryHandle, true); + + if (pCheckInfo->pDataCols == NULL) { + pCheckInfo->pDataCols = tdNewDataCols(1000, 2, 4096); + } + + tdInitDataCols(pCheckInfo->pDataCols, tsdbGetTableSchema(tsdbGetMeta(pQueryHandle->pTsdb), pCheckInfo->pTableObj)); + + SFile* pFile = &pQueryHandle->pFileGroup->files[TSDB_FILE_TYPE_DATA]; if (pFile->fd == FD_INITIALIZER) { pFile->fd = open(pFile->fname, O_RDONLY); } - - tsdbLoadDataBlock(pFile, pBlock, 1, pCheckInfo->pDataCols, data); - return true; + + if (tsdbLoadDataBlock(pFile, pBlock, 1, pCheckInfo->pDataCols, data) == 0) { + SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo; + + pBlockLoadInfo->fileGroup = pQueryHandle->pFileGroup; + pBlockLoadInfo->slot = pQueryHandle->cur.slot; + pBlockLoadInfo->sid = pCheckInfo->pTableObj->tableId.tid; + + blockLoaded = true; + } + + taosArrayDestroy(sa); + tfree(data); + + TSKEY* d = (TSKEY*)pCheckInfo->pDataCols->cols[PRIMARYKEY_TIMESTAMP_COL_INDEX].pData; + assert(d[0] == pBlock->keyFirst && d[pBlock->numOfPoints - 1] == pBlock->keyLast); + + return blockLoaded; } -static bool loadQualifiedDataFromFileBlock(STsdbQueryHandle *pQueryHandle) { - SQueryFilePos *cur = &pQueryHandle->cur; - - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); - SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; - - SArray *sa = getDefaultLoadColumns(pQueryHandle, true); +static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo) { + SArray* sa = getDefaultLoadColumns(pQueryHandle, true); + SQueryFilePos* cur = &pQueryHandle->cur; + if (QUERY_IS_ASC_QUERY(pQueryHandle->order)) { // query ended in current block if (pQueryHandle->window.ekey < pBlock->keyLast) { - doLoadDataFromFileBlock(pQueryHandle); - filterDataInDataBlock(pQueryHandle, pCheckInfo->pDataCols, sa); - } else { // the whole block is loaded in to buffer + if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) { + return false; + } + + SDataCols* pDataCols = pCheckInfo->pDataCols; + + if (pCheckInfo->lastKey > pBlock->keyFirst) { + cur->pos = + binarySearchForKey(pDataCols->cols[0].pData, pBlock->numOfPoints, pCheckInfo->lastKey, pQueryHandle->order); + } else { + cur->pos = 0; + } + + filterDataInDataBlock(pQueryHandle, pCheckInfo, pBlock, sa); + } else { // the whole block is loaded in to buffer pQueryHandle->realNumOfRows = pBlock->numOfPoints; } - } else {// todo desc query + } else { // todo desc query if (pQueryHandle->window.ekey > pBlock->keyFirst) { -// } } - + taosArrayDestroy(sa); return pQueryHandle->realNumOfRows > 0; } -bool moveToNextBlock(STsdbQueryHandle *pQueryHandle, int32_t step) { - SQueryFilePos *cur = &pQueryHandle->cur; - +bool moveToNextBlock(STsdbQueryHandle* pQueryHandle, int32_t step) { + SQueryFilePos* cur = &pQueryHandle->cur; + if (pQueryHandle->cur.fid >= 0) { /* * 1. ascending order. The last data block of data file @@ -405,74 +434,70 @@ bool moveToNextBlock(STsdbQueryHandle *pQueryHandle, int32_t step) { int32_t tid = pCheckInfo->tableId.tid; if ((step == QUERY_ASC_FORWARD_STEP && - (pQueryHandle->cur.slot == pCheckInfo->compIndex[tid].numOfSuperBlocks - 1)) || + (pQueryHandle->cur.slot == pQueryHandle->compIndex[tid].numOfSuperBlocks - 1)) || (step == QUERY_DESC_FORWARD_STEP && (pQueryHandle->cur.slot == 0))) { // temporarily keep the position value, in case of no data qualified when move forwards(backwards) -// SQueryFilePos save = pQueryHandle->cur; - SFileGroup* fgroup = tsdbGetFileGroupNext(&pCheckInfo->fileIter); + // SQueryFilePos save = pQueryHandle->cur; + pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter); int32_t fid = -1; - if (fgroup != NULL) { - if ((fid = getFileCompInfo(pCheckInfo, fgroup)) < 0) { + int32_t numOfBlocks = 0; + + if (pQueryHandle->pFileGroup != NULL) { + if ((fid = getFileCompInfo(pQueryHandle, &numOfBlocks, 1)) < 0) { } else { cur->slot = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pQueryHandle->numOfBlocks - 1; cur->pos = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pQueryHandle->pBlock[cur->slot].numOfPoints - 1; SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; - SCompData* data = calloc(1, sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols); - - data->numOfCols = pBlock->numOfCols; - data->uid = pCheckInfo->pTableObj->tableId.uid; - - cur->fid = fgroup->fileId; + cur->fid = pQueryHandle->pFileGroup->fileId; assert(cur->pos >= 0 && cur->fid >= 0 && cur->slot >= 0); if (pBlock->keyFirst > pQueryHandle->window.ekey) { // done return false; } - - loadQualifiedDataFromFileBlock(pQueryHandle); - return true; + + return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); } } else { // check data in cache pQueryHandle->cur.fid = -1; - return hasMoreDataInCacheForSingleModel(pQueryHandle); + return hasMoreDataInCache(pQueryHandle); } - } else { // next block in the same file + } else { // next block in the same file cur->slot += step; SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; cur->pos = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pBlock->numOfPoints - 1; - return loadQualifiedDataFromFileBlock(pQueryHandle); + return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); } } else { // data in cache - return hasMoreDataInCacheForSingleModel(pQueryHandle); + return hasMoreDataInCache(pQueryHandle); } - + return false; } -int vnodeBinarySearchKey(char *pValue, int num, TSKEY key, int order) { +int vnodeBinarySearchKey(char* pValue, int num, TSKEY key, int order) { int firstPos, lastPos, midPos = -1; int numOfPoints; - TSKEY *keyList; - + TSKEY* keyList; + if (num <= 0) return -1; - - keyList = (TSKEY *)pValue; + + keyList = (TSKEY*)pValue; firstPos = 0; lastPos = num - 1; - + if (order == 0) { // find the first position which is smaller than the key while (1) { if (key >= keyList[lastPos]) return lastPos; if (key == keyList[firstPos]) return firstPos; if (key < keyList[firstPos]) return firstPos - 1; - + numOfPoints = lastPos - firstPos + 1; midPos = (numOfPoints >> 1) + firstPos; - + if (key < keyList[midPos]) { lastPos = midPos - 1; } else if (key > keyList[midPos]) { @@ -481,13 +506,13 @@ int vnodeBinarySearchKey(char *pValue, int num, TSKEY key, int order) { break; } } - + } else { // find the first position which is bigger than the key while (1) { if (key <= keyList[firstPos]) return firstPos; if (key == keyList[lastPos]) return lastPos; - + if (key > keyList[lastPos]) { lastPos = lastPos + 1; if (lastPos >= num) @@ -495,10 +520,10 @@ int vnodeBinarySearchKey(char *pValue, int num, TSKEY key, int order) { else return lastPos; } - + numOfPoints = lastPos - firstPos + 1; midPos = (numOfPoints >> 1) + firstPos; - + if (key < keyList[midPos]) { lastPos = midPos - 1; } else if (key > keyList[midPos]) { @@ -508,20 +533,22 @@ int vnodeBinarySearchKey(char *pValue, int num, TSKEY key, int order) { } } } - + return midPos; } // only return the qualified data to client in terms of query time window, data rows in the same block but do not // be included in the query time window will be discarded -static void filterDataInDataBlock(STsdbQueryHandle *pQueryHandle, SDataCols* pCols, SArray *sa) { - SQueryFilePos *cur = &pQueryHandle->cur; - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); - SDataBlockInfo blockInfo = getTrueDataBlockInfo(pQueryHandle, pCheckInfo); - +static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock, + SArray* sa) { + SQueryFilePos* cur = &pQueryHandle->cur; + SDataBlockInfo blockInfo = getTrueDataBlockInfo(pCheckInfo, pBlock); + + SDataCols* pCols = pCheckInfo->pDataCols; + int32_t endPos = cur->pos; if (QUERY_IS_ASC_QUERY(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) { - endPos = blockInfo.size - 1; + endPos = blockInfo.rows - 1; pQueryHandle->realNumOfRows = endPos - cur->pos + 1; pCheckInfo->lastKey = blockInfo.window.ekey + 1; } else if (!QUERY_IS_ASC_QUERY(pQueryHandle->order) && pQueryHandle->window.ekey < blockInfo.window.skey) { @@ -529,8 +556,9 @@ static void filterDataInDataBlock(STsdbQueryHandle *pQueryHandle, SDataCols* pCo pQueryHandle->realNumOfRows = cur->pos + 1; pCheckInfo->lastKey = blockInfo.window.ekey - 1; } else { - endPos = vnodeBinarySearchKey(pCols->cols[0].pData, pCols->numOfPoints, pQueryHandle->window.ekey, pQueryHandle->order); - + endPos = + vnodeBinarySearchKey(pCols->cols[0].pData, pCols->numOfPoints, pQueryHandle->window.ekey, pQueryHandle->order); + if (QUERY_IS_ASC_QUERY(pQueryHandle->order)) { if (endPos < cur->pos) { pQueryHandle->realNumOfRows = 0; @@ -538,7 +566,7 @@ static void filterDataInDataBlock(STsdbQueryHandle *pQueryHandle, SDataCols* pCo } else { pQueryHandle->realNumOfRows = endPos - cur->pos; } - + pCheckInfo->lastKey = ((int64_t*)(pCols->cols[0].pData))[endPos] + 1; } else { if (endPos > cur->pos) { @@ -547,83 +575,84 @@ static void filterDataInDataBlock(STsdbQueryHandle *pQueryHandle, SDataCols* pCo } else { pQueryHandle->realNumOfRows = cur->pos - endPos; } - + assert(0); } } - + int32_t start = MIN(cur->pos, endPos); - + // move the data block in the front to data block if needed int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); - + for (int32_t i = 0; i < taosArrayGetSize(sa); ++i) { - int16_t colId = *(int16_t *)taosArrayGet(sa, i); - + int16_t colId = *(int16_t*)taosArrayGet(sa, i); + for (int32_t j = 0; j < numOfCols; ++j) { - SColumnInfoEx *pCol = taosArrayGet(pQueryHandle->pColumns, j); - + SColumnInfoEx* pCol = taosArrayGet(pQueryHandle->pColumns, j); + if (pCol->info.colId == colId) { SDataCol* pDataCol = &pCols->cols[i]; - memmove(pCol->pData, pDataCol->pData + pCol->info.bytes * start, pQueryHandle->realNumOfRows * pCol->info.bytes); + memmove(pCol->pData, pDataCol->pData + pCol->info.bytes * start, + pQueryHandle->realNumOfRows * pCol->info.bytes); break; } } } - - assert(pQueryHandle->realNumOfRows <= blockInfo.size); - + + assert(pQueryHandle->realNumOfRows <= blockInfo.rows); + // forward(backward) the position for cursor cur->pos = endPos; } -static SArray *getColumnIdList(STsdbQueryHandle *pQueryHandle) { +static SArray* getColumnIdList(STsdbQueryHandle* pQueryHandle) { int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); - SArray *pIdList = taosArrayInit(numOfCols, sizeof(int16_t)); + SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t)); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoEx *pCol = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoEx* pCol = taosArrayGet(pQueryHandle->pColumns, i); taosArrayPush(pIdList, &pCol->info.colId); } - + return pIdList; } -SArray *getDefaultLoadColumns(STsdbQueryHandle *pQueryHandle, bool loadTS) { - SArray *pLocalIdList = getColumnIdList(pQueryHandle); - +SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS) { + SArray* pLocalIdList = getColumnIdList(pQueryHandle); + // check if the primary time stamp column needs to load - int16_t colId = *(int16_t *)taosArrayGet(pLocalIdList, 0); - + int16_t colId = *(int16_t*)taosArrayGet(pLocalIdList, 0); + // the primary timestamp column does not be included in the the specified load column list, add it if (loadTS && colId != 0) { int16_t columnId = 0; taosArrayInsert(pLocalIdList, 0, &columnId); } - + return pLocalIdList; } -static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { +int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { int firstPos, lastPos, midPos = -1; int numOfPoints; - TSKEY *keyList; - + TSKEY* keyList; + if (num <= 0) return -1; - - keyList = (TSKEY *)pValue; + + keyList = (TSKEY*)pValue; firstPos = 0; lastPos = num - 1; - + if (order == 0) { // find the first position which is smaller than the key while (1) { if (key >= keyList[lastPos]) return lastPos; if (key == keyList[firstPos]) return firstPos; if (key < keyList[firstPos]) return firstPos - 1; - + numOfPoints = lastPos - firstPos + 1; midPos = (numOfPoints >> 1) + firstPos; - + if (key < keyList[midPos]) { lastPos = midPos - 1; } else if (key > keyList[midPos]) { @@ -632,13 +661,13 @@ static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { break; } } - + } else { // find the first position which is bigger than the key while (1) { if (key <= keyList[firstPos]) return firstPos; if (key == keyList[lastPos]) return lastPos; - + if (key > keyList[lastPos]) { lastPos = lastPos + 1; if (lastPos >= num) @@ -646,10 +675,10 @@ static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { else return lastPos; } - + numOfPoints = lastPos - firstPos + 1; midPos = (numOfPoints >> 1) + firstPos; - + if (key < keyList[midPos]) { lastPos = midPos - 1; } else if (key > keyList[midPos]) { @@ -659,513 +688,652 @@ static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { } } } - + return midPos; } -static bool getQualifiedDataBlock(STsdbQueryHandle *pQueryHandle, STableCheckInfo* pCheckInfo, int32_t type) { +static bool getQualifiedDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, int32_t type) { STsdbFileH* pFileHandle = tsdbGetFile(pQueryHandle->pTsdb); - int32_t fid = getFileIdFromKey(pCheckInfo->lastKey); - - tsdbInitFileGroupIter(pFileHandle, &pCheckInfo->fileIter, TSDB_FGROUP_ITER_FORWARD); - tsdbSeekFileGroupIter(&pCheckInfo->fileIter, fid); - pCheckInfo->pFileGroup = tsdbGetFileGroupNext(&pCheckInfo->fileIter); - + int32_t fid = getFileIdFromKey(pCheckInfo->lastKey); + + tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, TSDB_FGROUP_ITER_FORWARD); + tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid); + pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter); + SQueryFilePos* cur = &pQueryHandle->cur; - - TSKEY key = pCheckInfo->lastKey; - int32_t index = -1; int32_t tid = pCheckInfo->tableId.tid; - - while (pCheckInfo->pFileGroup != NULL) { - if (getFileCompInfo(pCheckInfo, pCheckInfo->pFileGroup) != TSDB_CODE_SUCCESS) { + int32_t numOfBlocks = 0; + + while (pQueryHandle->pFileGroup != NULL) { + if (getFileCompInfo(pQueryHandle, &numOfBlocks, 1) != TSDB_CODE_SUCCESS) { break; } - + + assert(pCheckInfo->numOfBlocks >= 0); + // no data block in current file, try next - if (pCheckInfo->compIndex[tid].numOfSuperBlocks == 0) { - dTrace("QInfo:%p no data block in file, fid:%d, tid:%d, try next", pQueryHandle->qinfo, - pCheckInfo->pFileGroup->fileId, tid); - - pCheckInfo->pFileGroup = tsdbGetFileGroupNext(&pCheckInfo->fileIter); - continue; - } - - index = binarySearchForBlockImpl(pCheckInfo->pCompInfo->blocks, pCheckInfo->compIndex[tid].numOfSuperBlocks, pQueryHandle->order, key); - - if (type == QUERY_RANGE_GREATER_EQUAL) { - if (key <= pCheckInfo->pCompInfo->blocks[index].keyLast) { - break; - } else { - index = -1; - } - } else { - if (key >= pCheckInfo->pCompInfo->blocks[index].keyFirst) { - break; - } else { - index = -1; - } + if (pCheckInfo->numOfBlocks > 0) { + cur->fid = pQueryHandle->pFileGroup->fileId; + break; } + + dTrace("%p no data block in file, fid:%d, tid:%d, try next, %p", pQueryHandle, pQueryHandle->pFileGroup->fileId, + tid, pQueryHandle->qinfo); + + pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter); } - - // failed to find qualified point in file, abort - if (index == -1) { + + if (pCheckInfo->numOfBlocks == 0) { return false; } - - assert(index >= 0 && index < pCheckInfo->compIndex[tid].numOfSuperBlocks); - - // load first data block into memory failed, caused by disk block error - bool blockLoaded = false; - SArray *sa = getDefaultLoadColumns(pQueryHandle, true); - - // todo no need to loaded at all - cur->slot = index; - + + cur->slot = 0; // always start from the first slot SCompBlock* pBlock = &pCheckInfo->pCompInfo->blocks[cur->slot]; - SCompData* data = calloc(1, sizeof(SCompData)+ sizeof(SCompCol)*pBlock->numOfCols); - - data->numOfCols = pBlock->numOfCols; - data->uid = pCheckInfo->pTableObj->tableId.uid; - - pCheckInfo->pDataCols = tdNewDataCols(1000, 2, 4096); - tdInitDataCols(pCheckInfo->pDataCols, pCheckInfo->pTableObj->schema); - - SFile* pFile = &pCheckInfo->pFileGroup->files[TSDB_FILE_TYPE_DATA]; - if (pFile->fd == FD_INITIALIZER) { - pFile->fd = open(pFile->fname, O_RDONLY); - } - - if (tsdbLoadDataBlock(pFile, pBlock, 1, pCheckInfo->pDataCols, data) == 0) { - SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo; - pBlockLoadInfo->fileGroup = pCheckInfo->pFileGroup; - pBlockLoadInfo->slot = pQueryHandle->cur.slot; - pBlockLoadInfo->sid = pCheckInfo->pTableObj->tableId.tid; - - blockLoaded = true; - } - - // dError("QInfo:%p fileId:%d total numOfBlks:%d blockId:%d load into memory failed due to error in disk files", - // GET_QINFO_ADDR(pQuery), pQuery->fileId, pQuery->numOfBlocks, blkIdx); - - // failed to load data from disk, abort current query - if (blockLoaded == false) { - taosArrayDestroy(sa); - tfree(data); - - return false; - } - - // todo search qualified points in blk, according to primary key (timestamp) column - SDataCols* pDataCols = pCheckInfo->pDataCols; - - TSKEY* d = (TSKEY*) pDataCols->cols[PRIMARYKEY_TIMESTAMP_COL_INDEX].pData; - assert(d[0] == pBlock->keyFirst && d[pBlock->numOfPoints - 1] == pBlock->keyLast); - - cur->pos = binarySearchForKey(pDataCols->cols[0].pData, pBlock->numOfPoints, key, pQueryHandle->order); - - cur->fid = pCheckInfo->pFileGroup->fileId; - assert(cur->pos >= 0 && cur->fid >= 0 && cur->slot >= 0); - - filterDataInDataBlock(pQueryHandle, pCheckInfo->pDataCols, sa); - - taosArrayDestroy(sa); - tfree(data); - return pQueryHandle->realNumOfRows > 0; + return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); } -static bool hasMoreDataInFileForSingleTableModel(STsdbQueryHandle* pHandle) { +static UNUSED_FUNC bool hasMoreDataForSingleTable(STsdbQueryHandle* pHandle) { assert(pHandle->activeIndex == 0 && taosArrayGetSize(pHandle->pTableCheckInfo) == 1); - - STsdbFileH* pFileHandle = tsdbGetFile(pHandle->pTsdb); + + STsdbFileH* pFileHandle = tsdbGetFile(pHandle->pTsdb); STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - + if (!pCheckInfo->checkFirstFileBlock) { pCheckInfo->checkFirstFileBlock = true; - + if (pFileHandle != NULL) { bool found = getQualifiedDataBlock(pHandle, pCheckInfo, 1); if (found) { return true; } } - + // no data in file, try cache pHandle->cur.fid = -1; - return hasMoreDataInCacheForSingleModel(pHandle); - } else { // move to next data block in file or in cache + return hasMoreDataInCache(pHandle); + } else { // move to next data block in file or in cache return moveToNextBlock(pHandle, 1); } } -static bool hasMoreDataInCacheForMultiModel(STsdbQueryHandle* pHandle) { - size_t numOfTables = taosArrayGetSize(pHandle->pTableCheckInfo); - assert(numOfTables > 0); - - while(pHandle->activeIndex < numOfTables) { - STableCheckInfo* pTableCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - - STable *pTable = pTableCheckInfo->pTableObj; - if (pTable->mem == NULL && pTable->imem == NULL) { - pHandle->activeIndex += 1; // try next table if exits - continue; +typedef struct SBlockOrderSupporter { + int32_t numOfTables; + STableBlockInfo** pDataBlockInfo; + int32_t* blockIndexArray; + int32_t* numOfBlocksPerMeter; +} SBlockOrderSupporter; + +void cleanBlockOrderSupporter(SBlockOrderSupporter* pSupporter, int32_t numOfTables) { + tfree(pSupporter->numOfBlocksPerMeter); + tfree(pSupporter->blockIndexArray); + + for (int32_t i = 0; i < numOfTables; ++i) { + tfree(pSupporter->pDataBlockInfo[i]); + } + + tfree(pSupporter->pDataBlockInfo); +} + +static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void* param) { + int32_t leftTableIndex = *(int32_t*)pLeft; + int32_t rightTableIndex = *(int32_t*)pRight; + + SBlockOrderSupporter* pSupporter = (SBlockOrderSupporter*)param; + + int32_t leftTableBlockIndex = pSupporter->blockIndexArray[leftTableIndex]; + int32_t rightTableBlockIndex = pSupporter->blockIndexArray[rightTableIndex]; + + if (leftTableBlockIndex > pSupporter->numOfBlocksPerMeter[leftTableIndex]) { + /* left block is empty */ + return 1; + } else if (rightTableBlockIndex > pSupporter->numOfBlocksPerMeter[rightTableIndex]) { + /* right block is empty */ + return -1; + } + + STableBlockInfo* pLeftBlockInfoEx = &pSupporter->pDataBlockInfo[leftTableIndex][leftTableBlockIndex]; + STableBlockInfo* pRightBlockInfoEx = &pSupporter->pDataBlockInfo[rightTableIndex][rightTableBlockIndex]; + + // assert(pLeftBlockInfoEx->pBlock.compBlock->offset != pRightBlockInfoEx->pBlock.compBlock->offset); + if (pLeftBlockInfoEx->pBlock.compBlock->offset == pRightBlockInfoEx->pBlock.compBlock->offset && + pLeftBlockInfoEx->pBlock.compBlock->last == pRightBlockInfoEx->pBlock.compBlock->last) { + // todo add more information + dError("error in header file, two block with same offset:%p", pLeftBlockInfoEx->pBlock.compBlock->offset); + } + + return pLeftBlockInfoEx->pBlock.compBlock->offset > pRightBlockInfoEx->pBlock.compBlock->offset ? 1 : -1; +} + +int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numOfBlocks, int32_t* numOfAllocBlocks) { + char* tmp = realloc(pQueryHandle->pDataBlockInfo, sizeof(STableBlockInfo) * numOfBlocks); + if (tmp == NULL) { + return TSDB_CODE_SERV_OUT_OF_MEMORY; + } + + pQueryHandle->pDataBlockInfo = (STableBlockInfo*)tmp; + memset(pQueryHandle->pDataBlockInfo, 0, sizeof(STableBlockInfo) * numOfBlocks); + *numOfAllocBlocks = numOfBlocks; + + int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + + SBlockOrderSupporter sup = {0}; + sup.numOfTables = numOfTables; + sup.numOfBlocksPerMeter = calloc(1, sizeof(int32_t) * numOfTables); + sup.blockIndexArray = calloc(1, sizeof(int32_t) * numOfTables); + sup.pDataBlockInfo = calloc(1, POINTER_BYTES * numOfTables); + + if (sup.numOfBlocksPerMeter == NULL || sup.blockIndexArray == NULL || sup.pDataBlockInfo == NULL) { + cleanBlockOrderSupporter(&sup, 0); + return TSDB_CODE_SERV_OUT_OF_MEMORY; + } + + int32_t cnt = 0; + int32_t numOfQualMeters = 0; + for (int32_t j = 0; j < numOfTables; ++j) { + STableCheckInfo* pTableCheck = (STableCheckInfo*)taosArrayGet(pQueryHandle->pTableCheckInfo, j); + + SCompBlock* pBlock = pTableCheck->pCompInfo->blocks; + sup.numOfBlocksPerMeter[numOfQualMeters] = pTableCheck->numOfBlocks; + + char* buf = calloc(1, sizeof(STableBlockInfo) * pTableCheck->numOfBlocks); + if (buf == NULL) { + cleanBlockOrderSupporter(&sup, numOfQualMeters); + return TSDB_CODE_SERV_OUT_OF_MEMORY; + } + + sup.pDataBlockInfo[numOfQualMeters] = (STableBlockInfo*)buf; + + for (int32_t k = 0; k < pTableCheck->numOfBlocks; ++k) { + STableBlockInfo* pBlockInfoEx = &sup.pDataBlockInfo[numOfQualMeters][k]; + + pBlockInfoEx->pBlock.compBlock = &pBlock[k]; + pBlockInfoEx->pBlock.fields = NULL; + + pBlockInfoEx->pTableCheckInfo = pTableCheck; + // pBlockInfoEx->groupIdx = pTableCheckInfo[j]->groupIdx; // set the group index + // pBlockInfoEx->blockIndex = pTableCheckInfo[j]->start + k; // set the block index in original meter + cnt++; + } + + numOfQualMeters++; + } + + dTrace("%p create data blocks info struct completed", pQueryHandle); + + assert(cnt <= numOfBlocks && numOfQualMeters <= numOfTables); // the pMeterDataInfo[j]->numOfBlocks may be 0 + sup.numOfTables = numOfQualMeters; + SLoserTreeInfo* pTree = NULL; + + uint8_t ret = tLoserTreeCreate(&pTree, sup.numOfTables, &sup, dataBlockOrderCompar); + if (ret != TSDB_CODE_SUCCESS) { + cleanBlockOrderSupporter(&sup, numOfTables); + return TSDB_CODE_SERV_OUT_OF_MEMORY; + } + + int32_t numOfTotal = 0; + + while (numOfTotal < cnt) { + int32_t pos = pTree->pNode[0].index; + int32_t index = sup.blockIndexArray[pos]++; + + STableBlockInfo* pBlocksInfoEx = sup.pDataBlockInfo[pos]; + pQueryHandle->pDataBlockInfo[numOfTotal++] = pBlocksInfoEx[index]; + + // set data block index overflow, in order to disable the offset comparator + if (sup.blockIndexArray[pos] >= sup.numOfBlocksPerMeter[pos]) { + sup.blockIndexArray[pos] = sup.numOfBlocksPerMeter[pos] + 1; } + + tLoserTreeAdjust(pTree, pos + sup.numOfTables); + } + + /* + * available when no import exists + * for(int32_t i = 0; i < cnt - 1; ++i) { + * assert((*pDataBlockInfo)[i].pBlock.compBlock->offset < (*pDataBlockInfo)[i+1].pBlock.compBlock->offset); + * } + */ + + dTrace("%p %d data blocks sort completed", pQueryHandle, cnt); + cleanBlockOrderSupporter(&sup, numOfTables); + free(pTree); + + return TSDB_CODE_SUCCESS; +} + +static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) { + STsdbFileH* pFileHandle = tsdbGetFile(pQueryHandle->pTsdb); + SQueryFilePos* cur = &pQueryHandle->cur; + + // find the start data block in file + if (!pQueryHandle->locateStart) { + pQueryHandle->locateStart = true; + + int32_t fid = getFileIdFromKey(pQueryHandle->window.skey); + + tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, TSDB_FGROUP_ITER_FORWARD); + tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid); + + int32_t numOfBlocks = -1; + + // todo opt for only one table case + pQueryHandle->numOfBlocks = 0; + int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + + while ((pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter)) != NULL) { + if (getFileCompInfo(pQueryHandle, &numOfBlocks, 1) != TSDB_CODE_SUCCESS) { + break; + } + + assert(numOfBlocks >= 0); + dTrace("%p %d blocks found in file for %d table(s), fid:%d", pQueryHandle, numOfBlocks, + pQueryHandle->pFileGroup->fileId, numOfTables); + + // todo return error code to query engine + if (createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks) != TSDB_CODE_SUCCESS) { + break; + } - // all data in mem are checked already. - if (pTableCheckInfo->lastKey > pTable->mem->keyLast) { - pHandle->activeIndex += 1; // try next table if exits - continue; + assert(numOfBlocks >= pQueryHandle->numOfBlocks); + if (pQueryHandle->numOfBlocks > 0) { + break; + } + } + + // no data in file anymore + if (pQueryHandle->numOfBlocks <= 0) { + assert(pQueryHandle->pFileGroup == NULL); + cur->fid = -1; + + return false; } + + cur->slot = 0; + cur->fid = pQueryHandle->pFileGroup->fileId; + + STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; + STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo; + SCompBlock* pBlock = pBlockInfo->pBlock.compBlock; - return true; - } + return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); + } else { + if (cur->slot == pQueryHandle->numOfBlocks - 1) { // all blocks + int32_t numOfBlocks = -1; + int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + pQueryHandle->numOfBlocks = 0; + + while ((pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter)) != NULL) { + if (getFileCompInfo(pQueryHandle, &numOfBlocks, 1) != TSDB_CODE_SUCCESS) { + break; + } + + assert(numOfBlocks >= 0); + dTrace("%p %d blocks found in file for %d table(s), fid:%d", pQueryHandle, numOfBlocks, numOfTables, + pQueryHandle->pFileGroup->fileId); - // all tables has checked already - return false; + // todo return error code to query engine + if (createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks) != TSDB_CODE_SUCCESS) { + break; + } + + assert(numOfBlocks >= pQueryHandle->numOfBlocks); + if (pQueryHandle->numOfBlocks > 0) { + break; + } + } + + // no data in file anymore + if (pQueryHandle->numOfBlocks <= 0) { + assert(pQueryHandle->pFileGroup == NULL); + cur->fid = -1; + return false; + } + + cur->slot = 0; + cur->fid = pQueryHandle->pFileGroup->fileId; + + STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; + + STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo; + SCompBlock* pBlock = pBlockInfo->pBlock.compBlock; + + return loadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); + } else { // next block of the same file + cur->slot += 1; + cur->pos = 0; + + STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; + return loadFileDataBlock(pQueryHandle, pBlockInfo->pBlock.compBlock, pBlockInfo->pTableCheckInfo); + } + } } // handle data in cache situation -bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) { - STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; - if (pHandle->model == SINGLE_TABLE_MODEL) { - return hasMoreDataInFileForSingleTableModel(pHandle); +bool tsdbNextDataBlock(tsdb_query_handle_t* pQueryHandle) { + STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle; + + size_t numOfTables = taosArrayGetSize(pHandle->pTableCheckInfo); + assert(numOfTables > 0); + + if (pHandle->checkFiles) { + if (getDataBlocksInFiles(pHandle)) { + return true; + } + + pHandle->activeIndex = 0; + pHandle->checkFiles = 0; + + while (pHandle->activeIndex < numOfTables) { + if (hasMoreDataInCache(pHandle)) { + return true; + } + + pHandle->activeIndex += 1; + } + + return false; } else { - return hasMoreDataInCacheForMultiModel(pHandle); + while (pHandle->activeIndex < numOfTables) { + if (hasMoreDataInCache(pHandle)) { + return true; + } + + pHandle->activeIndex += 1; + } + + return false; } } -static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, - TSKEY* skey, TSKEY* ekey, STsdbQueryHandle* pHandle) { - int numOfRows = 0; +static int tsdbReadRowsFromCache(SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, TSKEY* skey, TSKEY* ekey, + STsdbQueryHandle* pHandle) { + int numOfRows = 0; int32_t numOfCols = taosArrayGetSize(pHandle->pColumns); *skey = INT64_MIN; - - while(tSkipListIterNext(pIter)) { - SSkipListNode *node = tSkipListIterGet(pIter); + + while (tSkipListIterNext(pIter)) { + SSkipListNode* node = tSkipListIterGet(pIter); if (node == NULL) break; - + SDataRow row = SL_GET_NODE_DATA(node); if (dataRowKey(row) > maxKey) break; - + if (*skey == INT64_MIN) { *skey = dataRowKey(row); } - + *ekey = dataRowKey(row); - + int32_t offset = 0; - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoEx* pColInfo = taosArrayGet(pHandle->pColumns, i); - memcpy(pColInfo->pData + numOfRows*pColInfo->info.bytes, dataRowTuple(row) + offset, pColInfo->info.bytes); + memcpy(pColInfo->pData + numOfRows * pColInfo->info.bytes, dataRowTuple(row) + offset, pColInfo->info.bytes); offset += pColInfo->info.bytes; } - + numOfRows++; if (numOfRows >= maxRowsToRead) break; }; - + return numOfRows; } // copy data from cache into data block -SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) { - STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; - - STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - STable *pTable = pCheckInfo->pTableObj; - - TSKEY skey = 0, ekey = 0; +SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t* pQueryHandle) { + STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle; + + STable* pTable = NULL; + + TSKEY skey = 0, ekey = 0; int32_t rows = 0; - + // data in file - if (pHandle->cur.fid > 0) { - SDataBlockInfo binfo = getTrueDataBlockInfo(pHandle, pCheckInfo); - if (binfo.size == pHandle->realNumOfRows) { + if (pHandle->cur.fid >= 0) { + STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[pHandle->cur.slot]; + + pTable = pBlockInfo->pTableCheckInfo->pTableObj; + + SDataBlockInfo binfo = getTrueDataBlockInfo(pBlockInfo->pTableCheckInfo, pBlockInfo->pBlock.compBlock); + if (binfo.rows == pHandle->realNumOfRows) { + pBlockInfo->pTableCheckInfo->lastKey = pBlockInfo->pBlock.compBlock->keyLast + 1; return binfo; } else { /* not a whole disk block, only the qualified rows, so this block is loaded in to buffer during the * block next function */ SColumnInfoEx* pColInfoEx = taosArrayGet(pHandle->pColumns, 0); - + rows = pHandle->realNumOfRows; - skey = *(TSKEY*) pColInfoEx->pData; - ekey = *(TSKEY*) ((char*)pColInfoEx->pData + TSDB_KEYSIZE * (rows - 1)); + skey = *(TSKEY*)pColInfoEx->pData; + ekey = *(TSKEY*)((char*)pColInfoEx->pData + TSDB_KEYSIZE * (rows - 1)); + + // update the last key value + pBlockInfo->pTableCheckInfo->lastKey = ekey + 1; } } else { + STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); + pTable = pCheckInfo->pTableObj; + if (pTable->mem != NULL) { // create mem table iterator if it is not created yet if (pCheckInfo->iter == NULL) { pCheckInfo->iter = tSkipListCreateIter(pTable->mem->pData); } rows = tsdbReadRowsFromCache(pCheckInfo->iter, INT64_MAX, 2, &skey, &ekey, pHandle); + + // update the last key value + pCheckInfo->lastKey = ekey + 1; } } - + SDataBlockInfo blockInfo = { - .uid = pTable->tableId.uid, - .sid = pTable->tableId.tid, - .size = rows, - .window = {.skey = skey, .ekey = ekey} - }; - - // update the last key value - pCheckInfo->lastKey = ekey + 1; + .uid = pTable->tableId.uid, .sid = pTable->tableId.tid, .rows = rows, .window = {.skey = skey, .ekey = ekey}}; + return blockInfo; } // return null for data block in cache -int32_t tsdbRetrieveDataBlockStatisInfo(tsdb_query_handle_t *pQueryHandle, SDataStatis **pBlockStatis) { +int32_t tsdbRetrieveDataBlockStatisInfo(tsdb_query_handle_t* pQueryHandle, SDataStatis** pBlockStatis) { *pBlockStatis = NULL; return TSDB_CODE_SUCCESS; } -SArray *tsdbRetrieveDataBlock(tsdb_query_handle_t *pQueryHandle, SArray *pIdList) { +SArray* tsdbRetrieveDataBlock(tsdb_query_handle_t* pQueryHandle, SArray* pIdList) { /** * In the following two cases, the data has been loaded to SColumnInfoEx. * 1. data is from cache, 2. data block is not completed qualified to query time range */ - STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; - + STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle; + if (pHandle->cur.fid < 0) { return pHandle->pColumns; } else { - STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - - SDataBlockInfo binfo = getTrueDataBlockInfo(pHandle, pCheckInfo); - assert(pHandle->realNumOfRows <= binfo.size); - - if (pHandle->realNumOfRows < binfo.size) { + STableBlockInfo* pBlockInfoEx = &pHandle->pDataBlockInfo[pHandle->cur.slot]; + STableCheckInfo* pCheckInfo = pBlockInfoEx->pTableCheckInfo; + + SDataBlockInfo binfo = getTrueDataBlockInfo(pCheckInfo, pBlockInfoEx->pBlock.compBlock); + assert(pHandle->realNumOfRows <= binfo.rows); + + if (pHandle->realNumOfRows < binfo.rows) { return pHandle->pColumns; } else { - SArray *sa = getDefaultLoadColumns(pHandle, true); - // data block has been loaded, todo extract method SDataBlockLoadInfo* pBlockLoadInfo = &pHandle->dataBlockLoadInfo; if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->sid == pCheckInfo->pTableObj->tableId.tid) { return pHandle->pColumns; } else { - doLoadDataFromFileBlock(pHandle); - filterDataInDataBlock(pHandle, pCheckInfo->pDataCols, sa); - + SCompBlock* pBlock = pBlockInfoEx->pBlock.compBlock; + doLoadFileDataBlock(pHandle, pBlock, pCheckInfo); + + SArray* sa = getDefaultLoadColumns(pHandle, true); + filterDataInDataBlock(pHandle, pCheckInfo, pBlock, sa); + taosArrayDestroy(sa); + return pHandle->pColumns; } } } } -int32_t tsdbResetQuery(tsdb_query_handle_t *pQueryHandle, STimeWindow *window, tsdbpos_t position, int16_t order) { +int32_t tsdbResetQuery(tsdb_query_handle_t* pQueryHandle, STimeWindow* window, tsdbpos_t position, int16_t order) { return 0; } -int32_t tsdbDataBlockSeek(tsdb_query_handle_t *pQueryHandle, tsdbpos_t pos) { return 0;} +int32_t tsdbDataBlockSeek(tsdb_query_handle_t* pQueryHandle, tsdbpos_t pos) { return 0; } -tsdbpos_t tsdbDataBlockTell(tsdb_query_handle_t *pQueryHandle) { return NULL; } +tsdbpos_t tsdbDataBlockTell(tsdb_query_handle_t* pQueryHandle) { return NULL; } -SArray *tsdbRetrieveDataRow(tsdb_query_handle_t *pQueryHandle, SArray *pIdList, SQueryRowCond *pCond) { return NULL;} +SArray* tsdbRetrieveDataRow(tsdb_query_handle_t* pQueryHandle, SArray* pIdList, SQueryRowCond* pCond) { return NULL; } -tsdb_query_handle_t *tsdbQueryFromTagConds(STsdbQueryCond *pCond, int16_t stableId, const char *pTagFilterStr) { +tsdb_query_handle_t* tsdbQueryFromTagConds(STsdbQueryCond* pCond, int16_t stableId, const char* pTagFilterStr) { return NULL; } -SArray *tsdbGetTableList(tsdb_query_handle_t *pQueryHandle) { return NULL; } +SArray* tsdbGetTableList(tsdb_query_handle_t* pQueryHandle) { return NULL; } -static SArray* createTableIdArrayList(STsdbRepo* tsdb, int64_t uid) { +static int32_t getAllTableIdList(STsdbRepo* tsdb, int64_t uid, SArray* list) { STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); - assert(pTable != NULL); //assert pTable is a super table - - size_t size = tSkipListGetSize(pTable->pIndex); - SArray* pList = taosArrayInit(size, sizeof(STableId)); - + assert(pTable != NULL); // assert pTable is a super table + SSkipListIterator* iter = tSkipListCreateIter(pTable->pIndex); - while(tSkipListIterNext(iter)) { + while (tSkipListIterNext(iter)) { SSkipListNode* pNode = tSkipListIterGet(iter); - STable* t = *(STable**) SL_GET_NODE_DATA(pNode); - - taosArrayPush(pList, &t->tableId); + STable* t = *(STable**)SL_GET_NODE_DATA(pNode); + + taosArrayPush(list, &t->tableId); } - - return pList; + + return TSDB_CODE_SUCCESS; } -typedef struct SSyntaxTreeFilterSupporter { +typedef struct SExprTreeSupporter { SSchema* pTagSchema; int32_t numOfTags; int32_t optr; -} SSyntaxTreeFilterSupporter; +} SExprTreeSupporter; /** - * convert the result pointer to STabObj instead of tSkipListNode + * convert the result pointer to table id instead of table object pointer * @param pRes */ -static UNUSED_FUNC void tansformQueryResult(SArray* pRes) { - if (pRes == NULL || taosArrayGetSize(pRes) == 0) { +static void convertQueryResult(SArray* pRes, SArray* pTableList) { + if (pTableList == NULL || taosArrayGetSize(pTableList) == 0) { return; } - - size_t size = taosArrayGetSize(pRes); + + size_t size = taosArrayGetSize(pTableList); for (int32_t i = 0; i < size; ++i) { -// pRes->pRes[i] = ((tSkipListNode*)(pRes->pRes[i]))->pData; + STable* pTable = taosArrayGetP(pTableList, i); + taosArrayPush(pRes, &pTable->tableId); } } -void tSQLListTraverseDestroyInfo(void* param) { +void destroyHelper(void* param) { if (param == NULL) { return; } - + tQueryInfo* pInfo = (tQueryInfo*)param; tVariantDestroy(&(pInfo->q)); free(param); } -static char* convertTagQueryStr(const wchar_t* str, size_t len) { - char* mbs = NULL; - - if (len > 0) { - mbs = calloc(1, (len + 1) * TSDB_NCHAR_SIZE); - taosUcs4ToMbs((void*) str, len * TSDB_NCHAR_SIZE, mbs); //todo add log - } - - return mbs; -} - -static int32_t compareStrVal(const void* pLeft, const void* pRight) { - int32_t ret = strcmp(pLeft, pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } -} - -static int32_t compareWStrVal(const void* pLeft, const void* pRight) { - int32_t ret = wcscmp(pLeft, pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } -} - -static int32_t compareIntVal(const void* pLeft, const void* pRight) { - DEFAULT_COMP(GET_INT64_VAL(pLeft), GET_INT64_VAL(pRight)); -} - -static int32_t compareIntDoubleVal(const void* pLeft, const void* pRight) { - DEFAULT_COMP(GET_INT64_VAL(pLeft), GET_DOUBLE_VAL(pRight)); -} - -static int32_t compareDoubleVal(const void* pLeft, const void* pRight) { - DEFAULT_COMP(GET_DOUBLE_VAL(pLeft), GET_DOUBLE_VAL(pRight)); -} - -static int32_t compareDoubleIntVal(const void* pLeft, const void* pRight) { - double ret = (*(double*)pLeft) - (*(int64_t*)pRight); - if (fabs(ret) < DBL_EPSILON) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } -} - -static int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { +static UNUSED_FUNC int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { SPatternCompareInfo pInfo = {'%', '_'}; - + const char* pattern = pRight; const char* str = pLeft; - + int32_t ret = patternMatch(pattern, str, strlen(str), &pInfo); - + return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } -static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { +static UNUSED_FUNC int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { SPatternCompareInfo pInfo = {'%', '_'}; - + const wchar_t* pattern = pRight; const wchar_t* str = pLeft; - + int32_t ret = WCSPatternMatch(pattern, str, wcslen(str), &pInfo); - + return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } -static __compar_fn_t getFilterComparator(int32_t type, int32_t filterType, int32_t optr) { - __compar_fn_t comparator = NULL; - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_BOOL: { - if (filterType >= TSDB_DATA_TYPE_BOOL && filterType <= TSDB_DATA_TYPE_BIGINT) { - comparator = compareIntVal; - } else if (filterType >= TSDB_DATA_TYPE_FLOAT && filterType <= TSDB_DATA_TYPE_DOUBLE) { - comparator = compareIntDoubleVal; - } - break; - } - - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: { - if (filterType >= TSDB_DATA_TYPE_BOOL && filterType <= TSDB_DATA_TYPE_BIGINT) { - comparator = compareDoubleIntVal; - } else if (filterType >= TSDB_DATA_TYPE_FLOAT && filterType <= TSDB_DATA_TYPE_DOUBLE) { - comparator = compareDoubleVal; - } - break; - } - - case TSDB_DATA_TYPE_BINARY: { - assert(filterType == TSDB_DATA_TYPE_BINARY); - - if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ - comparator = compareStrPatternComp; - } else { /* normal relational comparator */ - comparator = compareStrVal; - } - - break; - } - - case TSDB_DATA_TYPE_NCHAR: { - assert(filterType == TSDB_DATA_TYPE_NCHAR); - - if (optr == TSDB_RELATION_LIKE) { - comparator = compareWStrPatternComp; - } else { - comparator = compareWStrVal; - } - - break; - } - default: - comparator = compareIntVal; - break; - } - - return comparator; -} +// static __compar_fn_t getFilterComparator(int32_t type, int32_t filterType, int32_t optr) { +// __compar_fn_t comparator = NULL; +// +// switch (type) { +// case TSDB_DATA_TYPE_TINYINT: +// case TSDB_DATA_TYPE_SMALLINT: +// case TSDB_DATA_TYPE_INT: +// case TSDB_DATA_TYPE_BIGINT: +// case TSDB_DATA_TYPE_BOOL: { +// if (filterType >= TSDB_DATA_TYPE_BOOL && filterType <= TSDB_DATA_TYPE_BIGINT) { +// comparator = compareIntVal; +// } else if (filterType >= TSDB_DATA_TYPE_FLOAT && filterType <= TSDB_DATA_TYPE_DOUBLE) { +// comparator = compareIntDoubleVal; +// } +// break; +// } +// +// case TSDB_DATA_TYPE_FLOAT: +// case TSDB_DATA_TYPE_DOUBLE: { +// if (filterType >= TSDB_DATA_TYPE_BOOL && filterType <= TSDB_DATA_TYPE_BIGINT) { +// comparator = compareDoubleIntVal; +// } else if (filterType >= TSDB_DATA_TYPE_FLOAT && filterType <= TSDB_DATA_TYPE_DOUBLE) { +// comparator = compareDoubleVal; +// } +// break; +// } +// +// case TSDB_DATA_TYPE_BINARY: { +// assert(filterType == TSDB_DATA_TYPE_BINARY); +// +// if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ +// comparator = compareStrPatternComp; +// } else { /* normal relational comparator */ +// comparator = compareStrVal; +// } +// +// break; +// } +// +// case TSDB_DATA_TYPE_NCHAR: { +// assert(filterType == TSDB_DATA_TYPE_NCHAR); +// +// if (optr == TSDB_RELATION_LIKE) { +// comparator = compareWStrPatternComp; +// } else { +// comparator = compareWStrVal; +// } +// +// break; +// } +// default: +// comparator = compareIntVal; +// break; +// } +// +// return comparator; +//} -static void getTagColumnInfo(SSyntaxTreeFilterSupporter* pSupporter, SSchema* pSchema, int32_t* index, - int32_t* offset) { +static void getTagColumnInfo(SExprTreeSupporter* pSupporter, SSchema* pSchema, int32_t* index, int32_t* offset) { *index = 0; *offset = 0; - + // filter on table name(TBNAME) if (strcasecmp(pSchema->name, TSQL_TBNAME_L) == 0) { *index = TSDB_TBNAME_COLUMN_INDEX; *offset = TSDB_TBNAME_COLUMN_INDEX; return; } - + while ((*index) < pSupporter->numOfTags) { if (pSupporter->pTagSchema[*index].bytes == pSchema->bytes && pSupporter->pTagSchema[*index].type == pSchema->type && - strcmp(pSupporter->pTagSchema[*index].name, pSchema->name) == 0) { + pSupporter->pTagSchema[*index].colId == pSchema->colId) { break; } else { (*offset) += pSupporter->pTagSchema[(*index)++].bytes; @@ -1174,53 +1342,53 @@ static void getTagColumnInfo(SSyntaxTreeFilterSupporter* pSupporter, SSchema* pS } void filterPrepare(void* expr, void* param) { - tExprNode *pExpr = (tExprNode*) expr; + tExprNode* pExpr = (tExprNode*)expr; if (pExpr->_node.info != NULL) { return; } - + int32_t i = 0, offset = 0; pExpr->_node.info = calloc(1, sizeof(tQueryInfo)); - + tQueryInfo* pInfo = pExpr->_node.info; - - SSyntaxTreeFilterSupporter* pSupporter = (SSyntaxTreeFilterSupporter*)param; - + + SExprTreeSupporter* pSupporter = (SExprTreeSupporter*)param; + tVariant* pCond = pExpr->_node.pRight->pVal; SSchema* pSchema = pExpr->_node.pLeft->pSchema; - + getTagColumnInfo(pSupporter, pSchema, &i, &offset); assert((i >= 0 && i < TSDB_MAX_TAGS) || (i == TSDB_TBNAME_COLUMN_INDEX)); assert((offset >= 0 && offset < TSDB_MAX_TAGS_LEN) || (offset == TSDB_TBNAME_COLUMN_INDEX)); - + pInfo->sch = *pSchema; - pInfo->colIdx = i; + pInfo->colIndex = i; pInfo->optr = pExpr->_node.optr; pInfo->offset = offset; - pInfo->compare = getFilterComparator(pSchema->type, pCond->nType, pInfo->optr); - + // pInfo->compare = getFilterComparator(pSchema->type, pCond->nType, pInfo->optr); + tVariantAssign(&pInfo->q, pCond); tVariantTypeSetType(&pInfo->q, pInfo->sch.type); } bool tSkipListNodeFilterCallback(const void* pNode, void* param) { tQueryInfo* pInfo = (tQueryInfo*)param; - + STable* pTable = (STable*)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); - - char* val = dataRowTuple(pTable->tagVal); // todo not only the first column + + char* val = dataRowTuple(pTable->tagVal); // todo not only the first column int8_t type = pInfo->sch.type; - + int32_t ret = 0; if (pInfo->q.nType == TSDB_DATA_TYPE_BINARY || pInfo->q.nType == TSDB_DATA_TYPE_NCHAR) { ret = pInfo->compare(val, pInfo->q.pz); } else { tVariant t = {0}; - tVariantCreateFromBinary(&t, val, (uint32_t) pInfo->sch.bytes, type); - + tVariantCreateFromBinary(&t, val, (uint32_t)pInfo->sch.bytes, type); + ret = pInfo->compare(&t.i64Key, &pInfo->q.i64Key); } - + switch (pInfo->optr) { case TSDB_RELATION_EQUAL: { return ret == 0; @@ -1228,10 +1396,10 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { case TSDB_RELATION_NOT_EQUAL: { return ret != 0; } - case TSDB_RELATION_LARGE_EQUAL: { + case TSDB_RELATION_GREATER_EQUAL: { return ret >= 0; } - case TSDB_RELATION_LARGE: { + case TSDB_RELATION_GREATER: { return ret > 0; } case TSDB_RELATION_LESS_EQUAL: { @@ -1243,90 +1411,85 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { case TSDB_RELATION_LIKE: { return ret == 0; } - + default: assert(false); } return true; } -static int32_t doQueryTableList(STable* pSTable, SArray* pRes, const char* pCond) { -// STColumn* stcol = schemaColAt(pSTable->tagSchema, 0); - - tExprNode* pExpr = NULL; -// tSQLBinaryExprFromString(&pExpr, stcol, schemaNCols(pSTable->tagSchema), (char*) pCond, strlen(pCond)); - - // failed to build expression, no result, return immediately - if (pExpr == NULL) { - mError("table:%" PRIu64 ", no result returned, error in super table query expression:%s", pSTable->tableId.uid, pCond); - tfree(pCond); - - return TSDB_CODE_OPS_NOT_SUPPORT; - } - +static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) { // query according to the binary expression -// SSyntaxTreeFilterSupporter s = {.pTagSchema = stcol, .numOfTags = schemaNCols(pSTable->tagSchema)}; -// -// SBinaryFilterSupp supp = { -// .fp = (__result_filter_fn_t)tSkipListNodeFilterCallback, -// .setupInfoFn = (__do_filter_suppl_fn_t)filterPrepare, -// .pExtInfo = &s -// }; -// -// tSQLBinaryExprTraverse(pExpr, pSTable->pIndex, pRes, &supp); -// tExprTreeDestroy(&pExpr, tSQLListTraverseDestroyInfo); -// -// tansformQueryResult(pRes); - + STSchema* pSchema = pSTable->tagSchema; + SSchema* schema = calloc(schemaNCols(pSchema), sizeof(SSchema)); + for (int32_t i = 0; i < schemaNCols(pSchema); ++i) { + schema[i].colId = schemaColAt(pSchema, i)->colId; + schema[i].type = schemaColAt(pSchema, i)->type; + schema[i].bytes = schemaColAt(pSchema, i)->bytes; + } + + SExprTreeSupporter s = {.pTagSchema = schema, .numOfTags = schemaNCols(pSTable->tagSchema)}; + + SBinaryFilterSupp supp = { + .fp = (__result_filter_fn_t)tSkipListNodeFilterCallback, .setupInfoFn = filterPrepare, .pExtInfo = &s}; + + SArray* pTableList = taosArrayInit(8, POINTER_BYTES); + + tExprTreeTraverse(pExpr, pSTable->pIndex, pTableList, &supp); + tExprTreeDestroy(&pExpr, destroyHelper); + + convertQueryResult(pRes, pTableList); return TSDB_CODE_SUCCESS; } -SArray *tsdbQueryTableList(tsdb_repo_t* tsdb, int64_t uid, const wchar_t *pTagCond, size_t len) { - // no condition, all tables created according to the stable will involved in querying - SArray* result = taosArrayInit(8, POINTER_BYTES); - - if (pTagCond == NULL || wcslen(pTagCond) == 0) { - return createTableIdArrayList(tsdb, uid); - } else { - char* str = convertTagQueryStr(pTagCond, len); - - STable* pSTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); - assert(pSTable != NULL); - - doQueryTableList(pSTable, result, str); - return result; +int32_t tsdbQueryTags(tsdb_repo_t* tsdb, int64_t uid, const char* pTagCond, size_t len, SArray* res) { + if (pTagCond == NULL || len == 0) { // no condition, all tables created according to this stable are involved + return getAllTableIdList(tsdb, uid, res); } + + STable* pSTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); + assert(pSTable != NULL); + + tExprNode* pExprNode = NULL; + int32_t ret = TSDB_CODE_SUCCESS; + + // failed to build expression, no result, return immediately + if ((ret = exprTreeFromBinary(pTagCond, len, &pExprNode) != TSDB_CODE_SUCCESS) || (pExprNode == NULL)) { + dError("stable:%" PRIu64 ", failed to deserialize expression tree, error exists", uid); + return ret; + } + + return doQueryTableList(pSTable, res, pExprNode); } void tsdbCleanupQueryHandle(tsdb_query_handle_t queryHandle) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) queryHandle; - + STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*)queryHandle; + size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - for(int32_t i = 0; i < size; ++i) { + for (int32_t i = 0; i < size; ++i) { STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); tSkipListDestroyIter(pTableCheckInfo->iter); if (pTableCheckInfo->pDataCols != NULL) { tfree(pTableCheckInfo->pDataCols->buf); } - + tfree(pTableCheckInfo->pDataCols); - + tfree(pTableCheckInfo->pCompInfo); - tfree(pTableCheckInfo->compIndex); } - + taosArrayDestroy(pQueryHandle->pTableCheckInfo); - + tfree(pQueryHandle->compIndex); + size_t cols = taosArrayGetSize(pQueryHandle->pColumns); - for(int32_t i = 0; i < cols; ++i) { - SColumnInfoEx *pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + for (int32_t i = 0; i < cols; ++i) { + SColumnInfoEx* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); tfree(pColInfo->pData); } - + taosArrayDestroy(pQueryHandle->pColumns); - tfree(pQueryHandle->unzipBuffer); - tfree(pQueryHandle->secondaryUnzipBuffer); + tfree(pQueryHandle->pDataBlockInfo); tfree(pQueryHandle); }