diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index f77897a74b77dfc9d168698bde927197eee8e9cc..405a29943b6512f2408b670bc86f4b1792523eea 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -39,7 +39,6 @@ extern "C" { #define UTIL_TABLE_IS_NORMAL_TABLE(metaInfo)\ (!(UTIL_TABLE_IS_SUPER_TABLE(metaInfo) || UTIL_TABLE_IS_CHILD_TABLE(metaInfo))) -#define TSDB_COL_IS_TAG(f) (((f)&TSDB_COL_TAG) != 0) typedef struct SParsedColElem { int16_t colIndex; diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index d2c52e972af5aa65bc97497f93936187fc97164f..c15212d99b4d14a75f4700c98efcb863a51ba397 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -468,7 +468,7 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField char* pData = pRes->data + pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row; // user defined constant value output columns - if (pInfo->pSqlExpr->colInfo.flag == TSDB_COL_UDC) { + if (TSDB_COL_IS_UD_COL(pInfo->pSqlExpr->colInfo.flag)) { if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { pData = pInfo->pSqlExpr->param[1].pz; pRes->length[columnIndex] = pInfo->pSqlExpr->param[1].nLen; diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index e74fcba24618a539843532d60c838d4565b3fb1e..cf256ee616d626db989d6b362001dfede0c898af 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -1648,8 +1648,9 @@ static void last_function(SQLFunctionCtx *pCtx) { for (int32_t i = pCtx->size - 1; i >= 0; --i) { char *data = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; + continue; } + memcpy(pCtx->aOutputBuf, data, pCtx->inputBytes); @@ -1721,7 +1722,9 @@ static void last_dist_function(SQLFunctionCtx *pCtx) { for (int32_t i = pCtx->size - 1; i >= 0; --i) { char *data = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; + if (!pCtx->requireNull) { + continue; + } } last_data_assign_impl(pCtx, data, i); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 5ce4c7125f3111cead393057e55e977e5cb6ede1..5cb08fb4a3bf5587cb325bc5cb07ce01982f4f38 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -33,6 +33,8 @@ #define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0" +#define TSWINDOW_IS_EQUAL(t1, t2) (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey)) + // -1 is tbname column index, so here use the -3 as the initial value #define COLUMN_INDEX_INITIAL_VAL (-3) #define COLUMN_INDEX_INITIALIZER \ @@ -45,6 +47,10 @@ typedef struct SColumnList { // todo refactor SColumnIndex ids[TSDB_MAX_COLUMNS]; } SColumnList; +typedef struct SConvertFunc { + int32_t originFuncId; + int32_t execFuncId; +} SConvertFunc; static SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t colIndex, int32_t tableIndex); static int32_t setShowInfo(SSqlObj* pSql, SSqlInfo* pInfo); @@ -1501,13 +1507,13 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t return TSDB_CODE_SUCCESS; } -static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSchema* pSchema, int32_t functionID, char* aliasName, +static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSchema* pSchema, SConvertFunc cvtFunc, char* aliasName, int32_t resColIdx, SColumnIndex* pColIndex) { int16_t type = 0; int16_t bytes = 0; - char columnName[TSDB_COL_NAME_LEN] = {0}; const char* msg1 = "not support column types"; + int32_t functionID = cvtFunc.execFuncId; if (functionID == TSDB_FUNC_SPREAD) { if (pSchema[pColIndex->columnIndex].type == TSDB_DATA_TYPE_BINARY || @@ -1523,16 +1529,21 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS type = pSchema[pColIndex->columnIndex].type; bytes = pSchema[pColIndex->columnIndex].bytes; } - + if (aliasName != NULL) { tstrncpy(columnName, aliasName, sizeof(columnName)); } else { - getRevisedName(columnName, functionID, sizeof(columnName) - 1, pSchema[pColIndex->columnIndex].name); + getRevisedName(columnName, cvtFunc.originFuncId, sizeof(columnName) - 1, pSchema[pColIndex->columnIndex].name); } + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, bytes, false); tstrncpy(pExpr->aliasName, columnName, sizeof(pExpr->aliasName)); + if (cvtFunc.originFuncId == TSDB_FUNC_LAST_ROW && cvtFunc.originFuncId != functionID) { + pExpr->colInfo.flag |= TSDB_COL_NULL; + } + // set reverse order scan data blocks for last query if (functionID == TSDB_FUNC_LAST) { pExpr->numOfParams = 1; @@ -1766,7 +1777,10 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (changeFunctionID(optr, &functionID) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9); } - + SConvertFunc cvtFunc = {.originFuncId = functionID, .execFuncId = functionID}; + if (functionID == TSDB_FUNC_LAST_ROW && TSWINDOW_IS_EQUAL(pQueryInfo->window,TSWINDOW_INITIALIZER)) { + cvtFunc.execFuncId = TSDB_FUNC_LAST; + } if (!requireAllFields) { if (pItem->pNode->pParam->nExpr < 1) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); @@ -1798,7 +1812,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col for (int32_t j = 0; j < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++j) { index.columnIndex = j; - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, functionID, pItem->aliasName, colIndex++, &index) != 0) { + if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex++, &index) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } } @@ -1815,8 +1829,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if ((index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) || (index.columnIndex < 0)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); } - - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, functionID, pItem->aliasName, colIndex + i, &index) != 0) { + if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex + i, &index) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -1853,7 +1866,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col for (int32_t i = 0; i < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++i) { SColumnIndex index = {.tableIndex = j, .columnIndex = i}; - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, functionID, pItem->aliasName, colIndex, &index) != 0) { + if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex, &index) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -5238,7 +5251,7 @@ static bool tagColumnInGroupby(SSqlGroupbyExpr* pGroupbyExpr, int16_t columnId) for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) { SColIndex* pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, j); - if (columnId == pColIndex->colId && pColIndex->flag == TSDB_COL_TAG) { + if (columnId == pColIndex->colId && TSDB_COL_IS_TAG(pColIndex->flag )) { return true; } } @@ -5537,7 +5550,6 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { return checkUpdateTagPrjFunctions(pQueryInfo, pCmd); } } - int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { const char* msg1 = "only one expression allowed"; const char* msg2 = "invalid expression in select clause"; @@ -6078,6 +6090,10 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { } int32_t joinQuery = (pQuerySql->from != NULL && pQuerySql->from->nExpr > 2); + + if (pQuerySql->pWhere) { + pQueryInfo->window = TSWINDOW_INITIALIZER; + } if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable, joinQuery) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 761a267ce53f9b2c54e0ea993ebd346564feee29..17e9c704f62984e15eb19e7876be554bea06dedb 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -170,6 +170,13 @@ enum _mgmt_table { #define TSDB_COL_NORMAL 0x0u // the normal column of the table #define TSDB_COL_TAG 0x1u // the tag column type #define TSDB_COL_UDC 0x2u // the user specified normal string column, it is a dummy column +#define TSDB_COL_NULL 0x4u // the column filter NULL or not + +#define TSDB_COL_IS_TAG(f) (((f&(~(TSDB_COL_NULL)))&TSDB_COL_TAG) != 0) +#define TSDB_COL_IS_NORMAL_COL(f) ((f&(~(TSDB_COL_NULL))) == TSDB_COL_NORMAL) +#define TSDB_COL_IS_UD_COL(f) ((f&(~(TSDB_COL_NULL))) == TSDB_COL_UDC) +#define TSDB_COL_REQ_NULL(f) (((f)&TSDB_COL_NULL) != 0) + extern char *taosMsg[]; diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 65ab82883b0a4a280e85bbaf66929081615795d3..384d8079a7533e68dea39ff7549f71ad4b748879 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -168,6 +168,7 @@ typedef struct SQLFunctionCtx { int16_t outputType; int16_t outputBytes; // size of results, determined by function and input column data type bool hasNull; // null value exist in current block + bool requireNull; // require null in some function int16_t functionId; // function id void * aInputElemBuf; char * aOutputBuf; // final result output buffer, point to sdata->data diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 4e2e31d2694712b14ed6d526a1ee960c0163398b..02112664cb0c2def9a6c1e06d2dd1e57388fd6f2 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -35,9 +35,7 @@ * forced to load primary column explicitly. */ #define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0) -#define TSDB_COL_IS_TAG(f) (((f)&TSDB_COL_TAG) != 0) -#define TSDB_COL_IS_NORMAL_COL(f) ((f) == TSDB_COL_NORMAL) -#define TSDB_COL_IS_UD_COL(f) ((f) == TSDB_COL_UDC) + #define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP) @@ -254,7 +252,7 @@ bool isGroupbyNormalCol(SSqlGroupbyExpr *pGroupbyExpr) { for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { SColIndex *pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, i); - if (pColIndex->flag == TSDB_COL_NORMAL) { + if (TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { //make sure the normal column locates at the second position if tbname exists in group by clause if (pGroupbyExpr->numOfGroupCols > 1) { assert(pColIndex->colIndex > 0); @@ -275,7 +273,7 @@ int16_t getGroupbyColumnType(SQuery *pQuery, SSqlGroupbyExpr *pGroupbyExpr) { for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { SColIndex *pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, i); - if (pColIndex->flag == TSDB_COL_NORMAL) { + if (TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { colId = pColIndex->colId; break; } @@ -1085,7 +1083,7 @@ static char *getGroupbyColumnData(SQuery *pQuery, int16_t *type, int16_t *bytes, for (int32_t k = 0; k < pGroupbyExpr->numOfGroupCols; ++k) { SColIndex* pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, k); - if (pColIndex->flag == TSDB_COL_TAG) { + if (TSDB_COL_IS_TAG(pColIndex->flag)) { continue; } @@ -1555,6 +1553,13 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; SColIndex* pIndex = &pSqlFuncMsg->colInfo; + if (TSDB_COL_REQ_NULL(pIndex->flag)) { + pCtx->requireNull = true; + pIndex->flag &= ~(TSDB_COL_NULL); + } else { + pCtx->requireNull = false; + } + int32_t index = pSqlFuncMsg->colInfo.colIndex; if (TSDB_COL_IS_TAG(pIndex->flag)) { if (pIndex->colId == TSDB_TBNAME_COLUMN_INDEX) { // todo refactor @@ -1574,6 +1579,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order pCtx->inputType = pQuery->colList[index].type; } + assert(isValidDataType(pCtx->inputType)); pCtx->ptsOutputBuf = NULL; @@ -1783,7 +1789,7 @@ static bool onlyQueryTags(SQuery* pQuery) { if (functionId != TSDB_FUNC_TAGPRJ && functionId != TSDB_FUNC_TID_TAG && (!(functionId == TSDB_FUNC_COUNT && pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX)) && - (!(functionId == TSDB_FUNC_PRJ && pExprInfo->base.colInfo.flag == TSDB_COL_UDC))) { + (!(functionId == TSDB_FUNC_PRJ && TSDB_COL_IS_UD_COL(pExprInfo->base.colInfo.flag)))) { return false; } } @@ -5353,7 +5359,7 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pE j += 1; } - } else if (pExprMsg->colInfo.flag == TSDB_COL_UDC) { // user specified column data + } else if (TSDB_COL_IS_UD_COL(pExprMsg->colInfo.flag)) { // user specified column data return TSDB_UD_COLUMN_INDEX; } else { while (j < pQueryMsg->numOfCols) { @@ -5561,7 +5567,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, int16_t functionId = pExprMsg->functionId; if (functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_TAG_DUMMY) { - if (pExprMsg->colInfo.flag != TSDB_COL_TAG) { // ignore the column index check for arithmetic expression. + if (!TSDB_COL_IS_TAG(pExprMsg->colInfo.flag)) { // ignore the column index check for arithmetic expression. code = TSDB_CODE_QRY_INVALID_MSG; goto _cleanup; } @@ -6843,7 +6849,7 @@ static void buildTagQueryResult(SQInfo* pQInfo) { int16_t type = 0, bytes = 0; for(int32_t j = 0; j < pQuery->numOfOutput; ++j) { // not assign value in case of user defined constant output column - if (pExprInfo[j].base.colInfo.flag == TSDB_COL_UDC) { + if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.colInfo.flag)) { continue; } diff --git a/src/query/src/qFill.c b/src/query/src/qFill.c index c1cfab3ea2c2ad9ed4d258f15930af8646d6f840..b13e32efbad9c501a778dab2986242e37b0d3eb0 100644 --- a/src/query/src/qFill.c +++ b/src/query/src/qFill.c @@ -55,7 +55,7 @@ SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_ SFillColInfo* pColInfo = &pFillInfo->pFillCol[i]; pFillInfo->pData[i] = calloc(1, pColInfo->col.bytes * capacity); - if (pColInfo->flag == TSDB_COL_TAG) { + if (TSDB_COL_IS_TAG(pColInfo->flag)) { bool exists = false; for(int32_t j = 0; j < k; ++j) { if (pFillInfo->pTags[j].col.colId == pColInfo->col.colId) { @@ -155,7 +155,7 @@ void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, tFilePage* pInpu char* data = pInput->data + pCol->col.offset * pInput->num; memcpy(pFillInfo->pData[i], data, (size_t)(pInput->num * pCol->col.bytes)); - if (pCol->flag == TSDB_COL_TAG) { // copy the tag value to tag value buffer + if (TSDB_COL_IS_TAG(pCol->flag)) { // copy the tag value to tag value buffer for (int32_t j = 0; j < pFillInfo->numOfTags; ++j) { SFillTagColInfo* pTag = &pFillInfo->pTags[j]; if (pTag->col.colId == pCol->col.colId) { @@ -251,7 +251,7 @@ int taosDoLinearInterpolation(int32_t type, SPoint* point1, SPoint* point2, SPoi static void setTagsValue(SFillInfo* pFillInfo, tFilePage** data, int32_t num) { for(int32_t j = 0; j < pFillInfo->numOfCols; ++j) { SFillColInfo* pCol = &pFillInfo->pFillCol[j]; - if (pCol->flag == TSDB_COL_NORMAL) { + if (TSDB_COL_IS_NORMAL_COL(pCol->flag)) { continue; } @@ -446,7 +446,7 @@ int32_t generateDataBlockImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t nu // assign rows to dst buffer for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (pCol->flag == TSDB_COL_TAG) { + if (TSDB_COL_IS_TAG(pCol->flag)) { continue; }