diff --git a/Jenkinsfile b/Jenkinsfile index 8d2429c137414fdb63260f94c0c99a6d264d8c48..e7603b578bf2c7718e67fbd9fda1674db49c07c0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -46,6 +46,7 @@ def pre_test(){ git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD develop)|grep -v -E '.*md|//src//connector|Jenkinsfile' || exit 0 + find ${WKC}/tests/pytest -name \'*\'.sql -exec rm -rf {} \\; cd ${WK} git reset --hard HEAD~10 git checkout develop @@ -115,7 +116,6 @@ pipeline { sh ''' date cd ${WKC}/tests - find pytest -name '*'sql|xargs rm -rf ./test-all.sh p1 date''' } @@ -131,7 +131,6 @@ pipeline { sh ''' date cd ${WKC}/tests - find pytest -name '*'sql|xargs rm -rf ./test-all.sh p2 date''' } diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 502d044d7540169497b682df96424ca3304f4668..f69ee23222119b3bc9b983c8fb24b067f11ff996 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -123,6 +123,7 @@ int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, i bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo); bool tscIsTWAQuery(SQueryInfo* pQueryInfo); bool tscIsSecondStageQuery(SQueryInfo* pQueryInfo); +bool tscGroupbyColumn(SQueryInfo* pQueryInfo); bool tscNonOrderedProjectionQueryOnSTable(SQueryInfo *pQueryInfo, int32_t tableIndex); bool tscOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex); @@ -133,6 +134,7 @@ bool tscIsProjectionQuery(SQueryInfo* pQueryInfo); bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex); bool tscQueryTags(SQueryInfo* pQueryInfo); bool tscMultiRoundQuery(SQueryInfo* pQueryInfo, int32_t tableIndex); +bool tscQueryBlockInfo(SQueryInfo* pQueryInfo); SSqlExpr* tscAddFuncInSelectClause(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, SSchema* pColSchema, int16_t colType); @@ -152,7 +154,6 @@ SInternalField* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_F SInternalField* tscFieldInfoGetInternalField(SFieldInfo* pFieldInfo, int32_t index); TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index); -void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo); void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo); int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 0bfbf3f946a4a1428549a493ab990a2be2173266..1740266518dd2c736feb47a5e9baa98cb66cbd94 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -198,9 +198,10 @@ typedef struct STableDataBlocks { typedef struct SQueryInfo { int16_t command; // the command may be different for each subclause, so keep it seperately. uint32_t type; // query/insert type + STimeWindow window; // the whole query time window - STimeWindow window; // query time window - SInterval interval; + SInterval interval; // tumble time window + SSessionWindow sessionWindow; // session time window SSqlGroupbyExpr groupbyExpr; // group by tags info SArray * colList; // SArray @@ -232,6 +233,7 @@ typedef struct SQueryInfo { typedef struct { int command; uint8_t msgType; + char reserve1[3]; // fix bus error on arm32 bool autoCreated; // create table if it is not existed during retrieve table meta in mnode union { @@ -244,8 +246,10 @@ typedef struct { char * curSql; // current sql, resume position of sql after parsing paused int8_t parseFinished; + char reserve2[3]; // fix bus error on arm32 int16_t numOfCols; + char reserve3[2]; // fix bus error on arm32 uint32_t allocSize; char * payload; int32_t payloadLen; @@ -255,7 +259,9 @@ typedef struct { int32_t numOfParams; int8_t dataSourceType; // load data from file or not + char reserve4[3]; // fix bus error on arm32 int8_t submitSchema; // submit block is built with table schema + char reserve5[3]; // fix bus error on arm32 STagData tagData; // NOTE: pTagData->data is used as a variant length array SName **pTableNameList; // all involved tableMeta list of current insert sql statement. @@ -397,7 +403,6 @@ typedef struct SSqlStream { void tscSetStreamDestTable(SSqlStream* pStream, const char* dstTable); - int tscAcquireRpc(const char *key, const char *user, const char *secret,void **pRpcObj); void tscReleaseRpc(void *param); void tscInitMsgsFp(); diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 23fb0ab67cded77ff737fac4246343486e80eb95..a44b0c46ba7920c2483c4baa3b34bc37f06bdee7 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -100,6 +100,10 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalMerger *pReducer, tOrderDescr } else if (functionId == TSDB_FUNC_APERCT) { pCtx->param[0].i64 = pExpr->param[0].i64; pCtx->param[0].nType = pExpr->param[0].nType; + } else if (functionId == TSDB_FUNC_BLKINFO) { + pCtx->param[0].i64 = pExpr->param[0].i64; + pCtx->param[0].nType = pExpr->param[0].nType; + pCtx->numOfParams = 1; } pCtx->interBufBytes = pExpr->interBytes; @@ -951,10 +955,10 @@ static void doFillResult(SSqlObj *pSql, SLocalMerger *pLocalMerge, bool doneOutp // todo extract function int64_t actualETime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey: pQueryInfo->window.skey; - tFilePage **pResPages = malloc(POINTER_BYTES * pQueryInfo->fieldsInfo.numOfOutput); + void** pResPages = malloc(POINTER_BYTES * pQueryInfo->fieldsInfo.numOfOutput); for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - pResPages[i] = calloc(1, sizeof(tFilePage) + pField->bytes * pLocalMerge->resColModel->capacity); + pResPages[i] = calloc(1, pField->bytes * pLocalMerge->resColModel->capacity); } while (1) { @@ -966,7 +970,7 @@ static void doFillResult(SSqlObj *pSql, SLocalMerger *pLocalMerge, bool doneOutp if (pQueryInfo->limit.offset > 0) { for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - memmove(pResPages[i]->data, pResPages[i]->data + pField->bytes * pQueryInfo->limit.offset, + memmove(pResPages[i], ((char*)pResPages[i]) + pField->bytes * pQueryInfo->limit.offset, (size_t)(newRows * pField->bytes)); } } @@ -1010,7 +1014,7 @@ static void doFillResult(SSqlObj *pSql, SLocalMerger *pLocalMerge, bool doneOutp int32_t offset = 0; for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i]->data, (size_t)(pField->bytes * pRes->numOfRows)); + memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i], (size_t)(pField->bytes * pRes->numOfRows)); offset += pField->bytes; } diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index a5906f5539a8d40b055f580933feb4b1184331f1..d071520b88b2d961e062766233295530fe48fd0f 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -307,7 +307,8 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SStrToken *pToken, char *payload, return tscInvalidSQLErrMsg(msg, "illegal float data", pToken->z); } - *((float *)payload) = (float)dv; +// *((float *)payload) = (float)dv; + SET_FLOAT_VAL(payload, dv); } break; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 2ff1775ecf38d6bd8a7207c9017a781c514251e3..a6298e631ff98bb48c98f0bb460c0453eb81ac74 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -75,10 +75,9 @@ static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int3 static int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pIdList, int16_t bytes, int8_t type, char* fieldName, SSqlExpr* pSqlExpr); -static int32_t convertFunctionId(int32_t optr, int16_t* functionId); static uint8_t convertOptr(SStrToken *pToken); -static int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery, bool intervalQuery); +static int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery, bool timeWindowQuery); static bool validateIpAddress(const char* ip, size_t size); static bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); @@ -87,8 +86,8 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery, bool static int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd); static int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); -static int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); -static int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); +static int32_t parseIntervalOffset(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStrToken* offsetToken); +static int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStrToken* pSliding); static int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExprItem* pItem); @@ -129,7 +128,11 @@ static int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo); static int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo); static int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index); static int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pSqlExpr, SQueryInfo* pQueryInfo, SArray* pCols, int64_t *uid); -static bool validateDebugFlag(int32_t flag); +static bool validateDebugFlag(int32_t v); + +static bool isTimeWindowQuery(SQueryInfo* pQueryInfo) { + return pQueryInfo->interval.interval > 0 || pQueryInfo->sessionWindow.gap > 0; +} int16_t getNewResColId(SQueryInfo* pQueryInfo) { return pQueryInfo->resColumnId--; @@ -176,11 +179,11 @@ static uint8_t convertOptr(SStrToken *pToken) { static bool validateDebugFlag(int32_t v) { const static int validFlag[] = {131, 135, 143}; - + for (int i = 0; i < tListLen(validFlag); i++) { if (v == validFlag[i]) { return true; - } + } } return false; } @@ -704,8 +707,8 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { SStrToken* t1 = taosArrayGet(pMiscInfo->a, 1); pCmd->payload[t->n] = ' '; // add sep strncpy(&pCmd->payload[t->n + 1], t1->z, t1->n); - } - return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_SUCCESS; } case TSDB_SQL_CREATE_TABLE: { @@ -822,21 +825,86 @@ static bool isTopBottomQuery(SQueryInfo* pQueryInfo) { return false; } -int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { +// need to add timestamp column in result set, if it is a time window query +static int32_t addPrimaryTsColumnForTimeWindowQuery(SQueryInfo* pQueryInfo) { + uint64_t uid = tscSqlExprGet(pQueryInfo, 0)->uid; + + int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL; + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); + if (pTableMetaInfo->pTableMeta->id.uid == uid) { + tableIndex = i; + break; + } + } + + if (tableIndex == COLUMN_INDEX_INITIAL_VAL) { + return TSDB_CODE_TSC_INVALID_SQL; + } + + SSchema s = {.bytes = TSDB_KEYSIZE, .type = TSDB_DATA_TYPE_TIMESTAMP, .colId = PRIMARYKEY_TIMESTAMP_COL_INDEX}; + tstrncpy(s.name, aAggs[TSDB_FUNC_TS].name, sizeof(s.name)); + + SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; + tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TS, &index, &s, TSDB_COL_NORMAL); + return TSDB_CODE_SUCCESS; +} + +static int32_t checkInvalidExprForTimeWindow(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { const char* msg1 = "invalid query expression"; + const char* msg2 = "top/bottom query does not support order by value in time window query"; + + // for top/bottom + interval query, we do not add additional timestamp column in the front + if (isTopBottomQuery(pQueryInfo)) { + + // invalid sql: + // top(col, k) from table_name [interval(1d)|session(ts, 1d)] order by k asc + // order by normal column is not supported + int32_t colId = pQueryInfo->order.orderColId; + if (isTimeWindowQuery(pQueryInfo) && colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + } + + return TSDB_CODE_SUCCESS; + } + + /* + * invalid sql: + * select count(tbname)/count(tag1)/count(tag2) from super_table_name [interval(1d)|session(ts, 1d)]; + */ + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < size; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + if (pExpr->functionId == TSDB_FUNC_COUNT && TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } + } + + /* + * invalid sql: + * select tbname, tags_fields from super_table_name [interval(1s)|session(ts,1s)] + */ + if (tscQueryTags(pQueryInfo) && isTimeWindowQuery(pQueryInfo)) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } + + return addPrimaryTsColumnForTimeWindowQuery(pQueryInfo); +} + +int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { const char* msg2 = "interval cannot be less than 10 ms"; const char* msg3 = "sliding cannot be used without interval"; - const char* msg4 = "top/bottom query does not support order by value in interval query"; SSqlCmd* pCmd = &pSql->cmd; STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - if (pQuerySql->interval.type == 0 || pQuerySql->interval.n == 0) { + if (pQuerySql->interval.interval.type == 0 || pQuerySql->interval.interval.n == 0) { if (pQuerySql->sliding.n > 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } + return TSDB_CODE_SUCCESS; } @@ -846,7 +914,7 @@ int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQ } // interval is not null - SStrToken* t = &pQuerySql->interval; + SStrToken* t = &pQuerySql->interval.interval; if (parseNatualDuration(t->z, t->n, &pQueryInfo->interval.interval, &pQueryInfo->interval.intervalUnit) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -863,78 +931,64 @@ int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQ } } - // for top/bottom + interval query, we do not add additional timestamp column in the front - if (isTopBottomQuery(pQueryInfo)) { - if (parseOffsetClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; - } + if (parseIntervalOffset(pCmd, pQueryInfo, &pQuerySql->interval.offset) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_SQL; + } - if (parseSlidingClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; - } + if (parseSlidingClause(pCmd, pQueryInfo, &pQuerySql->sliding) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_SQL; + } - int32_t colId = pQueryInfo->order.orderColId; - if (pQueryInfo->interval.interval > 0 && colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); - } + // The following part is used to check for the invalid query expression. + return checkInvalidExprForTimeWindow(pCmd, pQueryInfo); +} + +int32_t parseSessionClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL * pQuerySql) { + const char* msg1 = "gap should be fixed time window"; + const char* msg2 = "only one type time window allowed"; + const char* msg3 = "invalid column name"; + const char* msg4 = "invalid time window"; + // no session window + if (pQuerySql->sessionVal.gap.n == 0 || pQuerySql->sessionVal.col.n == 0) { return TSDB_CODE_SUCCESS; } - /* - * check invalid SQL: - * select count(tbname)/count(tag1)/count(tag2) from super_table_name interval(1d); - */ - size_t size = tscSqlExprNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < size; ++i) { - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); - if (pExpr->functionId == TSDB_FUNC_COUNT && TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); - } + SStrToken* col = &pQuerySql->sessionVal.col; + SStrToken* gap = &pQuerySql->sessionVal.gap; + + char timeUnit = 0; + if (parseNatualDuration(gap->z, gap->n, &pQueryInfo->sessionWindow.gap, &timeUnit) != TSDB_CODE_SUCCESS) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); } - /* - * check invalid SQL: - * select tbname, tags_fields from super_table_name interval(1s) - */ - if (tscQueryTags(pQueryInfo) && pQueryInfo->interval.interval > 0) { + if (timeUnit == 'y' || timeUnit == 'n') { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - // need to add timestamp column in result set, if interval is existed - uint64_t uid = tscSqlExprGet(pQueryInfo, 0)->uid; - - int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL; - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - if (pTableMetaInfo->pTableMeta->id.uid == uid) { - tableIndex = i; - break; - } + // if the unit of time window value is millisecond, change the value from microsecond + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); + if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { + pQueryInfo->sessionWindow.gap = pQueryInfo->sessionWindow.gap / 1000; } - if (tableIndex == COLUMN_INDEX_INITIAL_VAL) { - return TSDB_CODE_TSC_INVALID_SQL; + if (pQueryInfo->sessionWindow.gap != 0 && pQueryInfo->interval.interval != 0) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - SSchema s = {.bytes = TSDB_KEYSIZE, .type = TSDB_DATA_TYPE_TIMESTAMP, .colId = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tstrncpy(s.name, aAggs[TSDB_FUNC_TS].name, sizeof(s.name)); - - SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TS, &index, &s, TSDB_COL_NORMAL); - - if (parseOffsetClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + SColumnIndex index = COLUMN_INDEX_INITIALIZER; + if (getColumnIndexByName(pCmd, col, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } - if (parseSlidingClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; - } + pQueryInfo->sessionWindow.primaryColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; - return TSDB_CODE_SUCCESS; + // The following part is used to check for the invalid query expression. + return checkInvalidExprForTimeWindow(pCmd, pQueryInfo); } -int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { +int32_t parseIntervalOffset(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStrToken* offsetToken) { const char* msg1 = "interval offset cannot be negative"; const char* msg2 = "interval offset should be shorter than interval"; const char* msg3 = "cannot use 'year' as offset when interval is 'month'"; @@ -942,7 +996,7 @@ int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQue STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - SStrToken* t = &pQuerySql->offset; + SStrToken* t = offsetToken; if (t->n == 0) { pQueryInfo->interval.offsetUnit = pQueryInfo->interval.intervalUnit; pQueryInfo->interval.offset = 0; @@ -985,20 +1039,17 @@ int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQue return TSDB_CODE_SUCCESS; } -int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { +int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStrToken* pSliding) { const char* msg0 = "sliding value too small"; const char* msg1 = "sliding value no larger than the interval value"; const char* msg2 = "sliding value can not less than 1% of interval value"; const char* msg3 = "does not support sliding when interval is natural month/year"; -// const char* msg4 = "sliding not support yet in ordinary query"; const static int32_t INTERVAL_SLIDING_FACTOR = 100; - SSqlCmd* pCmd = &pSql->cmd; STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - SStrToken* pSliding = &pQuerySql->sliding; if (pSliding->n == 0) { pQueryInfo->interval.slidingUnit = pQueryInfo->interval.intervalUnit; pQueryInfo->interval.sliding = pQueryInfo->interval.interval; @@ -1090,7 +1141,7 @@ static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) { const char* msg1 = "first column must be timestamp"; const char* msg2 = "row length exceeds max length"; const char* msg3 = "duplicated column names"; - const char* msg4 = "invalid data types"; + const char* msg4 = "invalid data type"; const char* msg5 = "invalid binary/nchar column length"; const char* msg6 = "invalid column name"; @@ -1111,14 +1162,13 @@ static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) { int32_t nLen = 0; for (int32_t i = 0; i < numOfCols; ++i) { pField = taosArrayGet(pFieldList, i); - - if (pField->bytes == 0) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + if (!isValidDataType(pField->type)) { + invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); return false; } - if (!isValidDataType(pField->type)) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + if (pField->bytes == 0) { + invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); return false; } @@ -1311,7 +1361,7 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { const char* msg1 = "too many columns"; const char* msg2 = "duplicated column names"; const char* msg3 = "column length too long"; - const char* msg4 = "invalid data types"; + const char* msg4 = "invalid data type"; const char* msg5 = "invalid column name"; const char* msg6 = "invalid column length"; @@ -1657,10 +1707,11 @@ bool isValidDistinctSql(SQueryInfo* pQueryInfo) { } return false; } -int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery, bool intervalQuery) { + +int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery, bool timeWindowQuery) { assert(pSelection != NULL && pCmd != NULL); - const char* msg2 = "functions can not be mixed up"; + const char* msg2 = "functions or others can not be mixed up"; const char* msg3 = "not support query expression"; const char* msg5 = "invalid function name"; const char* msg6 = "only support distinct one tag"; @@ -1670,6 +1721,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel if (pQueryInfo->colList == NULL) { pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); } + bool hasDistinct = false; for (int32_t i = 0; i < pSelection->nExpr; ++i) { int32_t outputIndex = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); @@ -1678,26 +1730,25 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel if (hasDistinct == false) { hasDistinct = (pItem->distinct == true); } - // project on all fields - int32_t optr = pItem->pNode->nSQLOptr; - if (optr == TK_ALL || optr == TK_ID || optr == TK_STRING || optr == TK_INTEGER || optr == TK_FLOAT) { - // it is actually a function, but the function name is invalid - if (pItem->pNode->nSQLOptr == TK_ID && (pItem->pNode->colInfo.z == NULL && pItem->pNode->colInfo.n == 0)) { + int32_t type = pItem->pNode->type; + if (type == SQL_NODE_SQLFUNCTION) { + pItem->pNode->functionId = isValidFunction(pItem->pNode->operand.z, pItem->pNode->operand.n); + if (pItem->pNode->functionId < 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); } - // select table_name1.field_name1, table_name2.field_name2 from table_name1, table_name2 - if (addProjectionExprAndResultField(pCmd, pQueryInfo, pItem) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; - } - } else if (pItem->pNode->nSQLOptr >= TK_COUNT && pItem->pNode->nSQLOptr <= TK_TBID) { // sql function in selection clause, append sql function info in pSqlCmd structure sequentially if (addExprAndResultField(pCmd, pQueryInfo, outputIndex, pItem, true) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } - - } else if (pItem->pNode->nSQLOptr >= TK_PLUS && pItem->pNode->nSQLOptr <= TK_REM) { + } else if (type == SQL_NODE_TABLE_COLUMN || type == SQL_NODE_VALUE) { + // use the dynamic array list to decide if the function is valid or not + // select table_name1.field_name1, table_name2.field_name2 from table_name1, table_name2 + if (addProjectionExprAndResultField(pCmd, pQueryInfo, pItem) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_SQL; + } + } else if (type == SQL_NODE_EXPR) { int32_t code = handleArithmeticExpr(pCmd, clauseIndex, i, pItem); if (code != TSDB_CODE_SUCCESS) { return code; @@ -1720,11 +1771,11 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel // there is only one user-defined column in the final result field, add the timestamp column. size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList); - if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo)) { + if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo) && !tscQueryBlockInfo(pQueryInfo)) { addPrimaryTsColIntoResult(pQueryInfo); } - if (!functionCompatibleCheck(pQueryInfo, joinQuery, intervalQuery)) { + if (!functionCompatibleCheck(pQueryInfo, joinQuery, timeWindowQuery)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -1847,7 +1898,7 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t const char* msg1 = "tag for normal table query is not allowed"; int32_t startPos = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); - int32_t optr = pItem->pNode->nSQLOptr; + int32_t optr = pItem->pNode->tokenId; if (optr == TK_ALL) { // project on all fields TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY); @@ -1877,13 +1928,13 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t index.columnIndex = (pQueryInfo->udColumnId--); index.tableIndex = 0; - SSchema colSchema = tGetUserSpecifiedColumnSchema(&pItem->pNode->val, &pItem->pNode->token, pItem->aliasName); + SSchema colSchema = tGetUserSpecifiedColumnSchema(&pItem->pNode->value, &pItem->pNode->token, pItem->aliasName); SSqlExpr* pExpr = tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC); // NOTE: the first parameter is reserved for the tag column id during join query process. pExpr->numOfParams = 2; - tVariantAssign(&pExpr->param[1], &pItem->pNode->val); + tVariantAssign(&pExpr->param[1], &pItem->pNode->value); } else if (optr == TK_ID) { SColumnIndex index = COLUMN_INDEX_INITIALIZER; @@ -1992,9 +2043,25 @@ void setResultColName(char* name, tSqlExprItem* pItem, int32_t functionId, SStrT } } +static void updateLastScanOrderIfNeeded(SQueryInfo* pQueryInfo) { + if (pQueryInfo->sessionWindow.gap > 0 || tscGroupbyColumn(pQueryInfo)) { + size_t numOfExpr = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < numOfExpr; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + if (pExpr->functionId != TSDB_FUNC_LAST && pExpr->functionId != TSDB_FUNC_LAST_DST) { + continue; + } + + pExpr->numOfParams = 1; + pExpr->param->i64 = TSDB_ORDER_ASC; + pExpr->param->nType = TSDB_DATA_TYPE_INT; + } + } +} + int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSqlExprItem* pItem, bool finalResult) { STableMetaInfo* pTableMetaInfo = NULL; - int32_t optr = pItem->pNode->nSQLOptr; + int32_t functionId = pItem->pNode->functionId; const char* msg1 = "not support column types"; const char* msg2 = "invalid parameters"; @@ -2004,28 +2071,22 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col const char* msg6 = "function applied to tags not allowed"; const char* msg7 = "normal table can not apply this function"; const char* msg8 = "multi-columns selection does not support alias column name"; - const char* msg9 = "invalid function"; const char* msg10 = "diff can no be applied to unsigned numeric type"; - switch (optr) { - case TK_COUNT: { + switch (functionId) { + case TSDB_FUNC_COUNT: { /* more than one parameter for count() function */ if (pItem->pNode->pParam != NULL && pItem->pNode->pParam->nExpr != 1) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - int16_t functionID = 0; - if (convertFunctionId(optr, &functionID) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; - } - SSqlExpr* pExpr = NULL; SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (pItem->pNode->pParam != NULL) { tSqlExprItem* pParamElem = &pItem->pNode->pParam->a[0]; SStrToken* pToken = &pParamElem->pNode->colInfo; - int16_t sqlOptr = pParamElem->pNode->nSQLOptr; + int16_t sqlOptr = pParamElem->pNode->tokenId; if ((pToken->z == NULL || pToken->n == 0) && (TK_INTEGER != sqlOptr)) /*select count(1) from table*/ { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); @@ -2041,11 +2102,11 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; int32_t size = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; - pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, false); + pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, false); } else if (sqlOptr == TK_INTEGER) { // select count(1) from table1 char buf[8] = {0}; int64_t val = -1; - tVariant* pVariant = &pParamElem->pNode->val; + tVariant* pVariant = &pParamElem->pNode->value; if (pVariant->nType == TSDB_DATA_TYPE_BIGINT) { tVariantDump(pVariant, buf, TSDB_DATA_TYPE_BIGINT, true); val = GET_INT64_VAL(buf); @@ -2053,7 +2114,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (val == 1) { index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; int32_t size = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; - pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, false); + pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, false); } else { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -2073,12 +2134,12 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col } int32_t size = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; - pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, isTag); + pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, isTag); } } else { // count(*) is equalled to count(primary_timestamp_key) index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; int32_t size = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; - pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, false); + pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, false); } pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); @@ -2103,29 +2164,29 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col return TSDB_CODE_SUCCESS; } - case TK_SUM: - case TK_AVG: - case TK_RATE: - case TK_IRATE: - case TK_SUM_RATE: - case TK_SUM_IRATE: - case TK_AVG_RATE: - case TK_AVG_IRATE: - case TK_TWA: - case TK_MIN: - case TK_MAX: - case TK_DIFF: - case TK_STDDEV: - case TK_LEASTSQUARES: { + case TSDB_FUNC_SUM: + case TSDB_FUNC_AVG: + case TSDB_FUNC_RATE: + case TSDB_FUNC_IRATE: + case TSDB_FUNC_SUM_RATE: + case TSDB_FUNC_SUM_IRATE: + case TSDB_FUNC_AVG_RATE: + case TSDB_FUNC_AVG_IRATE: + case TSDB_FUNC_TWA: + case TSDB_FUNC_MIN: + case TSDB_FUNC_MAX: + case TSDB_FUNC_DIFF: + case TSDB_FUNC_STDDEV: + case TSDB_FUNC_LEASTSQR: { // 1. valid the number of parameters - if (pItem->pNode->pParam == NULL || (optr != TK_LEASTSQUARES && pItem->pNode->pParam->nExpr != 1) || - (optr == TK_LEASTSQUARES && pItem->pNode->pParam->nExpr != 3)) { + if (pItem->pNode->pParam == NULL || (functionId != TSDB_FUNC_LEASTSQR && pItem->pNode->pParam->nExpr != 1) || + (functionId == TSDB_FUNC_LEASTSQR && pItem->pNode->pParam->nExpr != 3)) { /* no parameters or more than one parameter for function */ return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } tSqlExprItem* pParamElem = &(pItem->pNode->pParam->a[0]); - if (pParamElem->pNode->nSQLOptr != TK_ALL && pParamElem->pNode->nSQLOptr != TK_ID) { + if (pParamElem->pNode->tokenId != TK_ALL && pParamElem->pNode->tokenId != TK_ID) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -2145,7 +2206,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (!IS_NUMERIC_TYPE(colType)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); - } else if (IS_UNSIGNED_NUMERIC_TYPE(colType) && optr == TK_DIFF) { + } else if (IS_UNSIGNED_NUMERIC_TYPE(colType) && functionId == TSDB_FUNC_DIFF) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg10); } @@ -2153,18 +2214,13 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col int16_t resultSize = 0; int32_t intermediateResSize = 0; - int16_t functionID = 0; - if (convertFunctionId(optr, &functionID) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; - } - - if (getResultDataInfo(pSchema->type, pSchema->bytes, functionID, 0, &resultType, &resultSize, + if (getResultDataInfo(pSchema->type, pSchema->bytes, functionId, 0, &resultType, &resultSize, &intermediateResSize, 0, false) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } // set the first column ts for diff query - if (optr == TK_DIFF) { + if (functionId == TSDB_FUNC_DIFF) { colIndex += 1; SColumnIndex indexTS = {.tableIndex = index.tableIndex, .columnIndex = 0}; SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &indexTS, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, @@ -2179,19 +2235,19 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); } - SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, resultType, resultSize, getNewResColId(pQueryInfo), resultSize, false); + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pQueryInfo), resultSize, false); - if (optr == TK_LEASTSQUARES) { + if (functionId == TSDB_FUNC_LEASTSQR) { /* set the leastsquares parameters */ char val[8] = {0}; - if (tVariantDump(&pParamElem[1].pNode->val, val, TSDB_DATA_TYPE_DOUBLE, true) < 0) { + if (tVariantDump(&pParamElem[1].pNode->value, val, TSDB_DATA_TYPE_DOUBLE, true) < 0) { return TSDB_CODE_TSC_INVALID_SQL; } addExprParams(pExpr, val, TSDB_DATA_TYPE_DOUBLE, DOUBLE_BYTES); memset(val, 0, tListLen(val)); - if (tVariantDump(&pParamElem[2].pNode->val, val, TSDB_DATA_TYPE_DOUBLE, true) < 0) { + if (tVariantDump(&pParamElem[2].pNode->value, val, TSDB_DATA_TYPE_DOUBLE, true) < 0) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -2217,21 +2273,16 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col tscInsertPrimaryTsSourceColumn(pQueryInfo, &index); return TSDB_CODE_SUCCESS; } - case TK_FIRST: - case TK_LAST: - case TK_SPREAD: - case TK_LAST_ROW: - case TK_INTERP: { + case TSDB_FUNC_FIRST: + case TSDB_FUNC_LAST: + case TSDB_FUNC_SPREAD: + case TSDB_FUNC_LAST_ROW: + case TSDB_FUNC_INTERP: { bool requireAllFields = (pItem->pNode->pParam == NULL); - int16_t functionID = 0; - if (convertFunctionId(optr, &functionID) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9); - } - // NOTE: has time range condition or normal column filter condition, the last_row query will be transferred to last query - SConvertFunc cvtFunc = {.originFuncId = functionID, .execFuncId = functionID}; - if (functionID == TSDB_FUNC_LAST_ROW && ((!TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER)) || (hasNormalColumnFilter(pQueryInfo)))) { + SConvertFunc cvtFunc = {.originFuncId = functionId, .execFuncId = functionId}; + if (functionId == TSDB_FUNC_LAST_ROW && ((!TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER)) || (hasNormalColumnFilter(pQueryInfo)))) { cvtFunc.execFuncId = TSDB_FUNC_LAST; } @@ -2247,13 +2298,13 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col /* in first/last function, multiple columns can be add to resultset */ for (int32_t i = 0; i < pItem->pNode->pParam->nExpr; ++i) { tSqlExprItem* pParamElem = &(pItem->pNode->pParam->a[i]); - if (pParamElem->pNode->nSQLOptr != TK_ALL && pParamElem->pNode->nSQLOptr != TK_ID) { + if (pParamElem->pNode->tokenId != TK_ALL && pParamElem->pNode->tokenId != TK_ID) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (pParamElem->pNode->nSQLOptr == TK_ALL) { // select table.* + if (pParamElem->pNode->tokenId == TK_ALL) { // select table.* SStrToken tmpToken = pParamElem->pNode->colInfo; if (getTableIndexByName(&tmpToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { @@ -2287,7 +2338,6 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col } char name[TSDB_COL_NAME_LEN] = {0}; - SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); bool multiColOutput = pItem->pNode->pParam->nExpr > 1; @@ -2296,22 +2346,6 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, name, colIndex + i, &index, finalResult) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } - - if (optr == TK_LAST) { // todo refactor - SSqlGroupbyExpr* pGroupBy = &pQueryInfo->groupbyExpr; - if (pGroupBy->numOfGroupCols > 0) { - for(int32_t k = 0; k < pGroupBy->numOfGroupCols; ++k) { - SColIndex* pIndex = taosArrayGet(pGroupBy->columnInfo, k); - if (!TSDB_COL_IS_TAG(pIndex->flag) && pIndex->colIndex < tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { // group by normal columns - SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, colIndex + i); - pExpr->numOfParams = 1; - pExpr->param->i64 = TSDB_ORDER_ASC; - - break; - } - } - } - } } } @@ -2338,21 +2372,20 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[index.columnIndex], cvtFunc, name, colIndex, &index, finalResult) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } - colIndex++; } numOfFields += tscGetNumOfColumns(pTableMetaInfo->pTableMeta); } - return TSDB_CODE_SUCCESS; } } - case TK_TOP: - case TK_BOTTOM: - case TK_PERCENTILE: - case TK_APERCENTILE: { + + case TSDB_FUNC_TOP: + case TSDB_FUNC_BOTTOM: + case TSDB_FUNC_PERCT: + case TSDB_FUNC_APERCT: { // 1. valid the number of parameters if (pItem->pNode->pParam == NULL || pItem->pNode->pParam->nExpr != 2) { /* no parameters or more than one parameter for function */ @@ -2360,7 +2393,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col } tSqlExprItem* pParamElem = &(pItem->pNode->pParam->a[0]); - if (pParamElem->pNode->nSQLOptr != TK_ID) { + if (pParamElem->pNode->tokenId != TK_ID) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -2388,11 +2421,11 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col } // 3. valid the parameters - if (pParamElem[1].pNode->nSQLOptr == TK_ID) { + if (pParamElem[1].pNode->tokenId == TK_ID) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - tVariant* pVariant = &pParamElem[1].pNode->val; + tVariant* pVariant = &pParamElem[1].pNode->value; int8_t resultType = pSchema[index.columnIndex].type; int16_t resultSize = pSchema[index.columnIndex].bytes; @@ -2400,7 +2433,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col char val[8] = {0}; SSqlExpr* pExpr = NULL; - if (optr == TK_PERCENTILE || optr == TK_APERCENTILE) { + if (functionId == TSDB_FUNC_PERCT || functionId == TSDB_FUNC_APERCT) { tVariantDump(pVariant, val, TSDB_DATA_TYPE_DOUBLE, true); double dp = GET_DOUBLE_VAL(val); @@ -2416,10 +2449,6 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col * for dp = 0, it is actually min, * for dp = 100, it is max, */ - int16_t functionId = 0; - if (convertFunctionId(optr, &functionId) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; - } tscInsertPrimaryTsSourceColumn(pQueryInfo, &index); colIndex += 1; // the first column is ts @@ -2433,11 +2462,6 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); } - int16_t functionId = 0; - if (convertFunctionId(optr, &functionId) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; - } - // todo REFACTOR // set the first column ts for top/bottom query SColumnIndex index1 = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; @@ -2471,7 +2495,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col return TSDB_CODE_SUCCESS; }; - case TK_TBID: { + case TSDB_FUNC_TID_TAG: { pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); @@ -2539,7 +2563,31 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col return TSDB_CODE_SUCCESS; } - + case TSDB_FUNC_BLKINFO: { + // no parameters or more than one parameter for function + if (pItem->pNode->pParam != NULL && pItem->pNode->pParam->nExpr != 0) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + } + + SColumnIndex index = {.tableIndex = 0, .columnIndex = TSDB_BLOCK_DIST_COLUMN_INDEX,}; + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + + SSchema s = {.name = "block_dist", .type = TSDB_DATA_TYPE_BINARY}; + int32_t inter = 0; + int16_t resType = 0; + int16_t bytes = 0; + getResultDataInfo(TSDB_DATA_TYPE_INT, 4, TSDB_FUNC_BLKINFO, 0, &resType, &bytes, &inter, 0, 0); + + s.bytes = bytes; + s.type = (uint8_t)resType; + SSqlExpr* pExpr = tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_BLKINFO, &index, &s, TSDB_COL_TAG); + pExpr->numOfParams = 1; + pExpr->param[0].i64 = pTableMetaInfo->pTableMeta->tableInfo.rowSize; + pExpr->param[0].nType = TSDB_DATA_TYPE_BIGINT; + + return TSDB_CODE_SUCCESS; + } + default: return TSDB_CODE_TSC_INVALID_SQL; } @@ -2564,8 +2612,8 @@ void getColumnName(tSqlExprItem* pItem, char* resultFieldName, int32_t nameLengt if (pItem->aliasName != NULL) { strncpy(resultFieldName, pItem->aliasName, nameLength); } else { - int32_t len = ((int32_t)pItem->pNode->operand.n < nameLength) ? (int32_t)pItem->pNode->operand.n : nameLength; - strncpy(resultFieldName, pItem->pNode->operand.z, len); + int32_t len = ((int32_t)pItem->pNode->token.n < nameLength) ? (int32_t)pItem->pNode->token.n : nameLength; + strncpy(resultFieldName, pItem->pNode->token.z, len); } } @@ -2705,87 +2753,6 @@ int32_t getColumnIndexByName(SSqlCmd* pCmd, const SStrToken* pToken, SQueryInfo* return doGetColumnIndexByName(pCmd, &tmpToken, pQueryInfo, pIndex); } -int32_t convertFunctionId(int32_t optr, int16_t* functionId) { - switch (optr) { - case TK_COUNT: - *functionId = TSDB_FUNC_COUNT; - break; - case TK_SUM: - *functionId = TSDB_FUNC_SUM; - break; - case TK_AVG: - *functionId = TSDB_FUNC_AVG; - break; - case TK_RATE: - *functionId = TSDB_FUNC_RATE; - break; - case TK_IRATE: - *functionId = TSDB_FUNC_IRATE; - break; - case TK_SUM_RATE: - *functionId = TSDB_FUNC_SUM_RATE; - break; - case TK_SUM_IRATE: - *functionId = TSDB_FUNC_SUM_IRATE; - break; - case TK_AVG_RATE: - *functionId = TSDB_FUNC_AVG_RATE; - break; - case TK_AVG_IRATE: - *functionId = TSDB_FUNC_AVG_IRATE; - break; - case TK_MIN: - *functionId = TSDB_FUNC_MIN; - break; - case TK_MAX: - *functionId = TSDB_FUNC_MAX; - break; - case TK_STDDEV: - *functionId = TSDB_FUNC_STDDEV; - break; - case TK_PERCENTILE: - *functionId = TSDB_FUNC_PERCT; - break; - case TK_APERCENTILE: - *functionId = TSDB_FUNC_APERCT; - break; - case TK_FIRST: - *functionId = TSDB_FUNC_FIRST; - break; - case TK_LAST: - *functionId = TSDB_FUNC_LAST; - break; - case TK_LEASTSQUARES: - *functionId = TSDB_FUNC_LEASTSQR; - break; - case TK_TOP: - *functionId = TSDB_FUNC_TOP; - break; - case TK_BOTTOM: - *functionId = TSDB_FUNC_BOTTOM; - break; - case TK_DIFF: - *functionId = TSDB_FUNC_DIFF; - break; - case TK_SPREAD: - *functionId = TSDB_FUNC_SPREAD; - break; - case TK_TWA: - *functionId = TSDB_FUNC_TWA; - break; - case TK_INTERP: - *functionId = TSDB_FUNC_INTERP; - break; - case TK_LAST_ROW: - *functionId = TSDB_FUNC_LAST_ROW; - break; - default: - return -1; - } - - return TSDB_CODE_SUCCESS; -} - int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { SSqlCmd* pCmd = &pSql->cmd; STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); @@ -3022,23 +2989,40 @@ bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) return false; } -static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery, bool intervalQuery) { +static bool groupbyTagsOrNull(SQueryInfo* pQueryInfo) { + if (pQueryInfo->groupbyExpr.columnInfo == NULL || + taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo) == 0) { + return true; + } + + size_t s = taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo); + for (int32_t i = 0; i < s; i++) { + SColIndex* colIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, i); + if (colIndex->flag != TSDB_COL_TAG) { + return false; + } + } + + return true; +} + +static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery, bool twQuery) { int32_t startIdx = 0; size_t numOfExpr = tscSqlExprNumOfExprs(pQueryInfo); assert(numOfExpr > 0); SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, startIdx); - int32_t functionID = pExpr->functionId; // ts function can be simultaneously used with any other functions. + int32_t functionID = pExpr->functionId; if (functionID == TSDB_FUNC_TS || functionID == TSDB_FUNC_TS_DUMMY) { startIdx++; } int32_t factor = functionCompatList[tscSqlExprGet(pQueryInfo, startIdx)->functionId]; - if (tscSqlExprGet(pQueryInfo, 0)->functionId == TSDB_FUNC_LAST_ROW && (joinQuery || intervalQuery)) { + if (tscSqlExprGet(pQueryInfo, 0)->functionId == TSDB_FUNC_LAST_ROW && (joinQuery || twQuery || !groupbyTagsOrNull(pQueryInfo))) { return false; } @@ -3066,7 +3050,7 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery, bool } } - if (functionId == TSDB_FUNC_LAST_ROW && (joinQuery || intervalQuery)) { + if (functionId == TSDB_FUNC_LAST_ROW && (joinQuery || twQuery || !groupbyTagsOrNull(pQueryInfo))) { return false; } } @@ -3219,39 +3203,39 @@ static int32_t doExtractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, colType = TSDB_DATA_TYPE_BIGINT; } else if (colType == TSDB_DATA_TYPE_FLOAT || colType == TSDB_DATA_TYPE_DOUBLE) { colType = TSDB_DATA_TYPE_DOUBLE; - } else if ((colType == TSDB_DATA_TYPE_TIMESTAMP) && (TSDB_DATA_TYPE_BINARY == pRight->val.nType)) { - int retVal = setColumnFilterInfoForTimestamp(pCmd, pQueryInfo, &pRight->val); + } else if ((colType == TSDB_DATA_TYPE_TIMESTAMP) && (TSDB_DATA_TYPE_BINARY == pRight->value.nType)) { + int retVal = setColumnFilterInfoForTimestamp(pCmd, pQueryInfo, &pRight->value); if (TSDB_CODE_SUCCESS != retVal) { return retVal; } } int32_t retVal = TSDB_CODE_SUCCESS; - if (pExpr->nSQLOptr == TK_LE || pExpr->nSQLOptr == TK_LT) { - retVal = tVariantDump(&pRight->val, (char*)&pColumnFilter->upperBndd, colType, false); + if (pExpr->tokenId == TK_LE || pExpr->tokenId == TK_LT) { + retVal = tVariantDump(&pRight->value, (char*)&pColumnFilter->upperBndd, colType, false); // TK_GT,TK_GE,TK_EQ,TK_NE are based on the pColumn->lowerBndd } else if (colType == TSDB_DATA_TYPE_BINARY) { - pColumnFilter->pz = (int64_t)calloc(1, pRight->val.nLen + TSDB_NCHAR_SIZE); - pColumnFilter->len = pRight->val.nLen; - retVal = tVariantDump(&pRight->val, (char*)pColumnFilter->pz, colType, false); + pColumnFilter->pz = (int64_t)calloc(1, pRight->value.nLen + TSDB_NCHAR_SIZE); + pColumnFilter->len = pRight->value.nLen; + retVal = tVariantDump(&pRight->value, (char*)pColumnFilter->pz, colType, false); } else if (colType == TSDB_DATA_TYPE_NCHAR) { - // pRight->val.nLen + 1 is larger than the actual nchar string length - pColumnFilter->pz = (int64_t)calloc(1, (pRight->val.nLen + 1) * TSDB_NCHAR_SIZE); - retVal = tVariantDump(&pRight->val, (char*)pColumnFilter->pz, colType, false); + // pRight->value.nLen + 1 is larger than the actual nchar string length + pColumnFilter->pz = (int64_t)calloc(1, (pRight->value.nLen + 1) * TSDB_NCHAR_SIZE); + retVal = tVariantDump(&pRight->value, (char*)pColumnFilter->pz, colType, false); size_t len = twcslen((wchar_t*)pColumnFilter->pz); pColumnFilter->len = len * TSDB_NCHAR_SIZE; } else { - retVal = tVariantDump(&pRight->val, (char*)&pColumnFilter->lowerBndd, colType, false); + retVal = tVariantDump(&pRight->value, (char*)&pColumnFilter->lowerBndd, colType, false); } if (retVal != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } - switch (pExpr->nSQLOptr) { + switch (pExpr->tokenId) { case TK_LE: pColumnFilter->upperRelOptr = TSDB_RELATION_LESS_EQUAL; break; @@ -3304,14 +3288,14 @@ typedef struct SCondExpr { static int32_t getTimeRange(STimeWindow* win, tSQLExpr* pRight, int32_t optr, int16_t timePrecision); static int32_t tSQLExprNodeToString(tSQLExpr* pExpr, char** str) { - if (pExpr->nSQLOptr == TK_ID) { // column name + if (pExpr->tokenId == TK_ID) { // column name strncpy(*str, pExpr->colInfo.z, pExpr->colInfo.n); *str += pExpr->colInfo.n; - } else if (pExpr->nSQLOptr >= TK_BOOL && pExpr->nSQLOptr <= TK_STRING) { // value - *str += tVariantToString(&pExpr->val, *str); + } else if (pExpr->tokenId >= TK_BOOL && pExpr->tokenId <= TK_STRING) { // value + *str += tVariantToString(&pExpr->value, *str); - } else if (pExpr->nSQLOptr >= TK_COUNT && pExpr->nSQLOptr <= TK_AVG_IRATE) { + } else if (pExpr->tokenId >= TSDB_FUNC_COUNT && pExpr->tokenId <= TSDB_FUNC_BLKINFO) { /* * arithmetic expression of aggregation, such as count(ts) + count(ts) *2 */ @@ -3324,10 +3308,10 @@ static int32_t tSQLExprNodeToString(tSQLExpr* pExpr, char** str) { return TSDB_CODE_SUCCESS; } -// pExpr->nSQLOptr == 0 while handling "is null" query +// pExpr->tokenId == 0 while handling "is null" query static bool isExprLeafNode(tSQLExpr* pExpr) { return (pExpr->pRight == NULL && pExpr->pLeft == NULL) && - (pExpr->nSQLOptr == 0 || pExpr->nSQLOptr == TK_ID || (pExpr->nSQLOptr >= TK_BOOL && pExpr->nSQLOptr <= TK_NCHAR) || pExpr->nSQLOptr == TK_SET); + (pExpr->tokenId == 0 || pExpr->tokenId == TK_ID || (pExpr->tokenId >= TK_BOOL && pExpr->tokenId <= TK_NCHAR) || pExpr->tokenId == TK_SET); } static bool isExprDirectParentOfLeafNode(tSQLExpr* pExpr) { @@ -3369,7 +3353,7 @@ static int32_t optrToString(tSQLExpr* pExpr, char** exprString) { const char* ne = "<>"; const char* likeOptr = "LIKE"; - switch (pExpr->nSQLOptr) { + switch (pExpr->tokenId) { case TK_LE: { *(int16_t*)(*exprString) = *(int16_t*)le; *exprString += 1; @@ -3436,13 +3420,13 @@ static int32_t tablenameListToString(tSQLExpr* pExpr, SStringBuilder* sb) { for (int32_t i = 0; i < pList->nExpr; ++i) { tSQLExpr* pSub = pList->a[i].pNode; - taosStringBuilderAppendStringLen(sb, pSub->val.pz, pSub->val.nLen); + taosStringBuilderAppendStringLen(sb, pSub->value.pz, pSub->value.nLen); if (i < pList->nExpr - 1) { taosStringBuilderAppendString(sb, TBNAME_LIST_SEP); } - if (pSub->val.nLen <= 0 || !tscValidateTableNameLength(pSub->val.nLen)) { + if (pSub->value.nLen <= 0 || !tscValidateTableNameLength(pSub->value.nLen)) { return TSDB_CODE_TSC_INVALID_SQL; } } @@ -3452,7 +3436,7 @@ static int32_t tablenameListToString(tSQLExpr* pExpr, SStringBuilder* sb) { static int32_t tablenameCondToString(tSQLExpr* pExpr, SStringBuilder* sb) { taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN); - taosStringBuilderAppendString(sb, pExpr->val.pz); + taosStringBuilderAppendString(sb, pExpr->value.pz); return TSDB_CODE_SUCCESS; } @@ -3506,21 +3490,21 @@ static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SC ((pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) ? 1 : 0); if (pColFilter->filterstr) { - if (pExpr->nSQLOptr != TK_EQ - && pExpr->nSQLOptr != TK_NE - && pExpr->nSQLOptr != TK_ISNULL - && pExpr->nSQLOptr != TK_NOTNULL - && pExpr->nSQLOptr != TK_LIKE + if (pExpr->tokenId != TK_EQ + && pExpr->tokenId != TK_NE + && pExpr->tokenId != TK_ISNULL + && pExpr->tokenId != TK_NOTNULL + && pExpr->tokenId != TK_LIKE ) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } } else { - if (pExpr->nSQLOptr == TK_LIKE) { + if (pExpr->tokenId == TK_LIKE) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (pSchema->type == TSDB_DATA_TYPE_BOOL) { - if (pExpr->nSQLOptr != TK_EQ && pExpr->nSQLOptr != TK_NE) { + if (pExpr->tokenId != TK_EQ && pExpr->tokenId != TK_NE) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } } @@ -3531,13 +3515,13 @@ static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SC } static void relToString(tSQLExpr* pExpr, char** str) { - assert(pExpr->nSQLOptr == TK_AND || pExpr->nSQLOptr == TK_OR); + assert(pExpr->tokenId == TK_AND || pExpr->tokenId == TK_OR); const char* or = "OR"; const char*and = "AND"; // if (pQueryInfo->tagCond.relType == TSQL_STABLE_QTYPE_COND) { - if (pExpr->nSQLOptr == TK_AND) { + if (pExpr->tokenId == TK_AND) { strcpy(*str, and); *str += strlen(and); } else { @@ -3591,10 +3575,10 @@ static int32_t getTablenameCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* int32_t ret = TSDB_CODE_SUCCESS; - if (pTableCond->nSQLOptr == TK_IN) { + if (pTableCond->tokenId == TK_IN) { ret = tablenameListToString(pRight, sb); - } else if (pTableCond->nSQLOptr == TK_LIKE) { - if (pRight->nSQLOptr != TK_STRING) { + } else if (pTableCond->tokenId == TK_LIKE) { + if (pRight->tokenId != TK_STRING) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -3614,12 +3598,12 @@ static int32_t getColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQ } if (!isExprDirectParentOfLeafNode(pExpr)) { // internal node - int32_t ret = getColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pLeft, pExpr->nSQLOptr); + int32_t ret = getColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pLeft, pExpr->tokenId); if (ret != TSDB_CODE_SUCCESS) { return ret; } - return getColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pRight, pExpr->nSQLOptr); + return getColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pRight, pExpr->tokenId); } else { // handle leaf node SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &pExpr->pLeft->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { @@ -3689,8 +3673,8 @@ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* } static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, - int32_t* type, uint64_t* uid) { - if (pExpr->nSQLOptr == TK_ID) { + int32_t* type, uint64_t* uid) { + if (pExpr->type == SQL_NODE_TABLE_COLUMN) { if (*type == NON_ARITHMEIC_EXPR) { *type = NORMAL_ARITHMETIC; } else if (*type == AGG_ARIGHTMEIC) { @@ -3712,9 +3696,9 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQuer } pList->ids[pList->num++] = index; - } else if (pExpr->nSQLOptr == TK_FLOAT && (isnan(pExpr->val.dKey) || isinf(pExpr->val.dKey))) { + } else if (pExpr->tokenId == TK_FLOAT && (isnan(pExpr->value.dKey) || isinf(pExpr->value.dKey))) { return TSDB_CODE_TSC_INVALID_SQL; - } else if (pExpr->nSQLOptr >= TK_COUNT && pExpr->nSQLOptr <= TK_AVG_IRATE) { + } else if (pExpr->type == SQL_NODE_SQLFUNCTION) { if (*type == NON_ARITHMEIC_EXPR) { *type = AGG_ARIGHTMEIC; } else if (*type == NORMAL_ARITHMETIC) { @@ -3727,6 +3711,11 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQuer // sql function list in selection clause. // Append the sqlExpr into exprList of pQueryInfo structure sequentially + pExpr->functionId = isValidFunction(pExpr->operand.z, pExpr->operand.n); + if (pExpr->functionId < 0) { + return TSDB_CODE_TSC_INVALID_SQL; + } + if (addExprAndResultField(pCmd, pQueryInfo, outputIndex, &item, false) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -3768,7 +3757,7 @@ static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryI uint64_t uidLeft = 0; uint64_t uidRight = 0; - if (pLeft->nSQLOptr >= TK_PLUS && pLeft->nSQLOptr <= TK_REM) { + if (pLeft->type == SQL_NODE_EXPR) { int32_t ret = validateArithmeticSQLExpr(pCmd, pLeft, pQueryInfo, pList, type); if (ret != TSDB_CODE_SUCCESS) { return ret; @@ -3781,7 +3770,7 @@ static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryI } tSQLExpr* pRight = pExpr->pRight; - if (pRight->nSQLOptr >= TK_PLUS && pRight->nSQLOptr <= TK_REM) { + if (pRight->type == SQL_NODE_EXPR) { int32_t ret = validateArithmeticSQLExpr(pCmd, pRight, pQueryInfo, pList, type); if (ret != TSDB_CODE_SUCCESS) { return ret; @@ -3814,22 +3803,15 @@ static bool isValidExpr(tSQLExpr* pLeft, tSQLExpr* pRight, int32_t optr) { * * However, columnA < 4+12 is valid */ - if (pLeft->nSQLOptr >= TK_COUNT && pLeft->nSQLOptr <= TK_AVG_IRATE) { + if (pLeft->type == SQL_NODE_SQLFUNCTION) { return false; } if (pRight == NULL) { return true; } - - if (pRight->nSQLOptr >= TK_COUNT && pRight->nSQLOptr <= TK_AVG_IRATE) { - return false; - } - if (pLeft->nSQLOptr >= TK_BOOL - && pLeft->nSQLOptr <= TK_BINARY - && pRight->nSQLOptr >= TK_BOOL - && pRight->nSQLOptr <= TK_BINARY) { + if (pLeft->tokenId >= TK_BOOL && pLeft->tokenId <= TK_BINARY && pRight->tokenId >= TK_BOOL && pRight->tokenId <= TK_BINARY) { return false; } @@ -3840,8 +3822,8 @@ static void exchangeExpr(tSQLExpr* pExpr) { tSQLExpr* pLeft = pExpr->pLeft; tSQLExpr* pRight = pExpr->pRight; - if (pRight->nSQLOptr == TK_ID && (pLeft->nSQLOptr == TK_INTEGER || pLeft->nSQLOptr == TK_FLOAT || - pLeft->nSQLOptr == TK_STRING || pLeft->nSQLOptr == TK_BOOL)) { + if (pRight->tokenId == TK_ID && (pLeft->tokenId == TK_INTEGER || pLeft->tokenId == TK_FLOAT || + pLeft->tokenId == TK_STRING || pLeft->tokenId == TK_BOOL)) { /* * exchange value of the left handside and the value of the right-handside * to make sure that the value of filter expression always locates in @@ -3849,7 +3831,7 @@ static void exchangeExpr(tSQLExpr* pExpr) { * the column-id is at the left handside. */ uint32_t optr = 0; - switch (pExpr->nSQLOptr) { + switch (pExpr->tokenId) { case TK_LE: optr = TK_GE; break; @@ -3863,10 +3845,10 @@ static void exchangeExpr(tSQLExpr* pExpr) { optr = TK_LE; break; default: - optr = pExpr->nSQLOptr; + optr = pExpr->tokenId; } - pExpr->nSQLOptr = optr; + pExpr->tokenId = optr; SWAP(pExpr->pLeft, pExpr->pRight, void*); } } @@ -3880,11 +3862,11 @@ static bool validateJoinExprNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr tSQLExpr* pRight = pExpr->pRight; - if (pRight->nSQLOptr != TK_ID) { + if (pRight->tokenId != TK_ID) { return true; } - if (pExpr->nSQLOptr != TK_EQ) { + if (pExpr->tokenId != TK_EQ) { invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); return false; } @@ -3926,7 +3908,7 @@ static bool validTableNameOptr(tSQLExpr* pExpr) { const char nameFilterOptr[] = {TK_IN, TK_LIKE}; for (int32_t i = 0; i < tListLen(nameFilterOptr); ++i) { - if (pExpr->nSQLOptr == nameFilterOptr[i]) { + if (pExpr->tokenId == nameFilterOptr[i]) { return true; } } @@ -3980,7 +3962,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQL } // set join query condition - if (pRight->nSQLOptr == TK_ID) { // no need to keep the timestamp join condition + if (pRight->tokenId == TK_ID) { // no need to keep the timestamp join condition TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY); pCondExpr->tsJoin = true; @@ -4002,8 +3984,8 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQL } // check for like expression - if ((*pExpr)->nSQLOptr == TK_LIKE) { - if (pRight->val.nLen > TSDB_PATTERN_STRING_MAX_LEN) { + if ((*pExpr)->tokenId == TK_LIKE) { + if (pRight->value.nLen > TSDB_PATTERN_STRING_MAX_LEN) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg8); } @@ -4036,7 +4018,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQL *type = TSQL_EXPR_TBNAME; *pExpr = NULL; } else { - if (pRight != NULL && pRight->nSQLOptr == TK_ID) { // join on tag columns for stable query + if (pRight != NULL && pRight->tokenId == TK_ID) { // join on tag columns for stable query if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -4060,7 +4042,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQL } else { // query on other columns *type = TSQL_EXPR_COLUMN; - if (pRight->nSQLOptr == TK_ID) { // other column cannot be served as the join column + if (pRight->tokenId == TK_ID) { // other column cannot be served as the join column return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); } @@ -4082,7 +4064,7 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr** pExpr tSQLExpr* pLeft = (*pExpr)->pLeft; tSQLExpr* pRight = (*pExpr)->pRight; - if (!isValidExpr(pLeft, pRight, (*pExpr)->nSQLOptr)) { + if (!isValidExpr(pLeft, pRight, (*pExpr)->tokenId)) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -4090,12 +4072,12 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr** pExpr int32_t rightType = -1; if (!isExprDirectParentOfLeafNode(*pExpr)) { - int32_t ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pLeft, pCondExpr, &leftType, (*pExpr)->nSQLOptr); + int32_t ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pLeft, pCondExpr, &leftType, (*pExpr)->tokenId); if (ret != TSDB_CODE_SUCCESS) { return ret; } - ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pRight, pCondExpr, &rightType, (*pExpr)->nSQLOptr); + ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pRight, pCondExpr, &rightType, (*pExpr)->tokenId); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -4105,7 +4087,7 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr** pExpr * expression is not valid for parent node, it must be TK_AND operator. */ if (leftType != rightType) { - if ((*pExpr)->nSQLOptr == TK_OR && (leftType + rightType != TSQL_EXPR_TBNAME + TSQL_EXPR_TAG)) { + if ((*pExpr)->tokenId == TK_OR && (leftType + rightType != TSQL_EXPR_TBNAME + TSQL_EXPR_TAG)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } } @@ -4133,7 +4115,7 @@ static void doCompactQueryExpr(tSQLExpr** pExpr) { } if ((*pExpr)->pLeft == NULL && (*pExpr)->pRight == NULL && - ((*pExpr)->nSQLOptr == TK_OR || (*pExpr)->nSQLOptr == TK_AND)) { + ((*pExpr)->tokenId == TK_OR || (*pExpr)->tokenId == TK_AND)) { tSqlExprNodeDestroy(*pExpr); *pExpr = NULL; @@ -4167,7 +4149,7 @@ static void doExtractExprForSTable(SSqlCmd* pCmd, tSQLExpr** pExpr, SQueryInfo* (*pExpr) = NULL; } else { - *pOut = tSqlExprCreate(NULL, NULL, (*pExpr)->nSQLOptr); + *pOut = tSqlExprCreate(NULL, NULL, (*pExpr)->tokenId); doExtractExprForSTable(pCmd, &(*pExpr)->pLeft, pQueryInfo, &((*pOut)->pLeft), tableIndex); doExtractExprForSTable(pCmd, &(*pExpr)->pRight, pQueryInfo, &((*pOut)->pRight), tableIndex); @@ -4211,9 +4193,9 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, STagCond* pTagCond = &pQueryInfo->tagCond; pTagCond->tbnameCond.uid = pTableMetaInfo->pTableMeta->id.uid; - assert(pExpr->nSQLOptr == TK_LIKE || pExpr->nSQLOptr == TK_IN); + assert(pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_IN); - if (pExpr->nSQLOptr == TK_LIKE) { + if (pExpr->tokenId == TK_LIKE) { char* str = taosStringBuilderGetResult(sb, NULL); pQueryInfo->tagCond.tbnameCond.cond = strdup(str); pQueryInfo->tagCond.tbnameCond.len = (int32_t) strlen(str); @@ -4309,7 +4291,7 @@ static int32_t getTimeRangeFromExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLE } if (!isExprDirectParentOfLeafNode(pExpr)) { - if (pExpr->nSQLOptr == TK_OR) { + if (pExpr->tokenId == TK_OR) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -4328,7 +4310,7 @@ static int32_t getTimeRangeFromExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLE tSQLExpr* pRight = pExpr->pRight; STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX}; - if (getTimeRange(&win, pRight, pExpr->nSQLOptr, tinfo.precision) != TSDB_CODE_SUCCESS) { + if (getTimeRange(&win, pRight, pExpr->tokenId, tinfo.precision) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); } @@ -4472,7 +4454,7 @@ static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) { retVal = tVariantDump(vVariant, tmp, schemaType, false); free(tmp); } else if (schemaType == TSDB_DATA_TYPE_NCHAR) { - // pRight->val.nLen + 1 is larger than the actual nchar string length + // pRight->value.nLen + 1 is larger than the actual nchar string length char *tmp = calloc(1, (vVariant->nLen + 1) * TSDB_NCHAR_SIZE); retVal = tVariantDump(vVariant, tmp, schemaType, false); free(tmp); @@ -4571,7 +4553,7 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql } int32_t type = 0; - if ((ret = getQueryCondExpr(&pSql->cmd, pQueryInfo, pExpr, &condExpr, &type, (*pExpr)->nSQLOptr)) != TSDB_CODE_SUCCESS) { + if ((ret = getQueryCondExpr(&pSql->cmd, pQueryInfo, pExpr, &condExpr, &type, (*pExpr)->tokenId)) != TSDB_CODE_SUCCESS) { return ret; } @@ -4628,7 +4610,7 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql int32_t getTimeRange(STimeWindow* win, tSQLExpr* pRight, int32_t optr, int16_t timePrecision) { // this is join condition, do nothing - if (pRight->nSQLOptr == TK_ID) { + if (pRight->tokenId == TK_ID) { return TSDB_CODE_SUCCESS; } @@ -4636,42 +4618,42 @@ int32_t getTimeRange(STimeWindow* win, tSQLExpr* pRight, int32_t optr, int16_t t * filter primary ts filter expression like: * where ts in ('2015-12-12 4:8:12') */ - if (pRight->nSQLOptr == TK_SET || optr == TK_IN) { + if (pRight->tokenId == TK_SET || optr == TK_IN) { return TSDB_CODE_TSC_INVALID_SQL; } int64_t val = 0; bool parsed = false; - if (pRight->val.nType == TSDB_DATA_TYPE_BINARY) { - pRight->val.nLen = strdequote(pRight->val.pz); + if (pRight->value.nType == TSDB_DATA_TYPE_BINARY) { + pRight->value.nLen = strdequote(pRight->value.pz); - char* seg = strnchr(pRight->val.pz, '-', pRight->val.nLen, false); + char* seg = strnchr(pRight->value.pz, '-', pRight->value.nLen, false); if (seg != NULL) { - if (taosParseTime(pRight->val.pz, &val, pRight->val.nLen, TSDB_TIME_PRECISION_MICRO, tsDaylight) == TSDB_CODE_SUCCESS) { + if (taosParseTime(pRight->value.pz, &val, pRight->value.nLen, TSDB_TIME_PRECISION_MICRO, tsDaylight) == TSDB_CODE_SUCCESS) { parsed = true; } else { return TSDB_CODE_TSC_INVALID_SQL; } } else { - SStrToken token = {.z = pRight->val.pz, .n = pRight->val.nLen, .type = TK_ID}; - int32_t len = tSQLGetToken(pRight->val.pz, &token.type); + SStrToken token = {.z = pRight->value.pz, .n = pRight->value.nLen, .type = TK_ID}; + int32_t len = tSQLGetToken(pRight->value.pz, &token.type); - if ((token.type != TK_INTEGER && token.type != TK_FLOAT) || len != pRight->val.nLen) { + if ((token.type != TK_INTEGER && token.type != TK_FLOAT) || len != pRight->value.nLen) { return TSDB_CODE_TSC_INVALID_SQL; } } - } else if (pRight->nSQLOptr == TK_INTEGER && timePrecision == TSDB_TIME_PRECISION_MILLI) { + } else if (pRight->tokenId == TK_INTEGER && timePrecision == TSDB_TIME_PRECISION_MILLI) { /* - * if the pRight->nSQLOptr == TK_INTEGER/TK_FLOAT, the value is adaptive, we + * if the pRight->tokenId == TK_INTEGER/TK_FLOAT, the value is adaptive, we * need the time precision in metermeta to transfer the value in MICROSECOND * * Additional check to avoid data overflow */ - if (pRight->val.i64 <= INT64_MAX / 1000) { - pRight->val.i64 *= 1000; + if (pRight->value.i64 <= INT64_MAX / 1000) { + pRight->value.i64 *= 1000; } - } else if (pRight->nSQLOptr == TK_FLOAT && timePrecision == TSDB_TIME_PRECISION_MILLI) { - pRight->val.dKey *= 1000; + } else if (pRight->tokenId == TK_FLOAT && timePrecision == TSDB_TIME_PRECISION_MILLI) { + pRight->value.dKey *= 1000; } if (!parsed) { @@ -4679,7 +4661,7 @@ int32_t getTimeRange(STimeWindow* win, tSQLExpr* pRight, int32_t optr, int16_t t * failed to parse timestamp in regular formation, try next * it may be a epoch time in string format */ - tVariantDump(&pRight->val, (char*)&val, TSDB_DATA_TYPE_BIGINT, true); + tVariantDump(&pRight->value, (char*)&val, TSDB_DATA_TYPE_BIGINT, true); /* * transfer it into MICROSECOND format if it is a string, since for @@ -4687,7 +4669,7 @@ int32_t getTimeRange(STimeWindow* win, tSQLExpr* pRight, int32_t optr, int16_t t * * additional check to avoid data overflow */ - if (pRight->nSQLOptr == TK_STRING && timePrecision == TSDB_TIME_PRECISION_MILLI) { + if (pRight->tokenId == TK_STRING && timePrecision == TSDB_TIME_PRECISION_MILLI) { if (val <= INT64_MAX / 1000) { val *= 1000; } @@ -5498,7 +5480,7 @@ int32_t validateLocalConfig(SMiscInfo* pOptions) { // reset log does not need value for (int32_t i = 0; i < 1; ++i) { SDNodeDynConfOption* pOption = &LOCAL_DYNAMIC_CFG_OPTIONS[i]; - if ((pOption->len == pOptionToken->n) && + if ((pOption->len == pOptionToken->n) && (strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0)) { return TSDB_CODE_SUCCESS; } @@ -5513,7 +5495,7 @@ int32_t validateLocalConfig(SMiscInfo* pOptions) { for (int32_t i = 1; i < tListLen(LOCAL_DYNAMIC_CFG_OPTIONS); ++i) { SDNodeDynConfOption* pOption = &LOCAL_DYNAMIC_CFG_OPTIONS[i]; - if ((pOption->len == pOptionToken->n) + if ((pOption->len == pOptionToken->n) && (strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0)) { return TSDB_CODE_SUCCESS; } @@ -6098,8 +6080,8 @@ static int32_t doAddGroupbyColumnsOnDemand(SSqlCmd* pCmd, SQueryInfo* pQueryInfo SColumnList ids = getColumnList(1, 0, pColIndex->colIndex); insertResultField(pQueryInfo, (int32_t)size, &ids, bytes, (int8_t)type, name, pExpr); } else { - // if this query is "group by" normal column, interval is not allowed - if (pQueryInfo->interval.interval > 0) { + // if this query is "group by" normal column, time window query is not allowed + if (isTimeWindowQuery(pQueryInfo)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -6161,7 +6143,7 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); } - if (pQueryInfo->groupbyExpr.numOfGroupCols > 0 || pQueryInfo->interval.interval > 0) { + if (pQueryInfo->groupbyExpr.numOfGroupCols > 0 || isTimeWindowQuery(pQueryInfo)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); } else { return TSDB_CODE_SUCCESS; @@ -6263,8 +6245,8 @@ int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ index = 2; } else { for (int32_t i = 0; i < tListLen(functionsInfo); ++i) { - if (strncasecmp(functionsInfo[i].name, pExpr->operand.z, functionsInfo[i].len) == 0 && - functionsInfo[i].len == pExpr->operand.n) { + if (strncasecmp(functionsInfo[i].name, pExpr->token.z, functionsInfo[i].len) == 0 && + functionsInfo[i].len == pExpr->token.n) { index = i; break; } @@ -6559,12 +6541,12 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { tVariantListItem* pItem = taosArrayGet(pValList, i); findColumnIndex = false; - + // todo speedup by using hash list for (int32_t t = 0; t < schemaSize; ++t) { if (strncmp(sToken->z, pTagSchema[t].name, sToken->n) == 0 && strlen(pTagSchema[t].name) == sToken->n) { SSchema* pSchema = &pTagSchema[t]; - + char tagVal[TSDB_MAX_TAGS_LEN]; if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { if (pItem->pVar.nLen > pSchema->bytes) { @@ -6572,9 +6554,9 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } } - + ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true); - + // check again after the convert since it may be converted from binary to nchar. if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { int16_t len = varDataTLen(tagVal); @@ -6583,12 +6565,12 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } } - + if (ret != TSDB_CODE_SUCCESS) { tdDestroyKVRowBuilder(&kvRowBuilder); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); } - + tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal); findColumnIndex = true; @@ -6599,7 +6581,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { if (!findColumnIndex) { tdDestroyKVRowBuilder(&kvRowBuilder); return tscInvalidSQLErrMsg(pCmd->payload, "invalid tag name", sToken->z); - } + } } } else { if (schemaSize != valSize) { @@ -6610,7 +6592,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { for (int32_t i = 0; i < valSize; ++i) { SSchema* pSchema = &pTagSchema[i]; tVariantListItem* pItem = taosArrayGet(pValList, i); - + char tagVal[TSDB_MAX_TAGS_LEN]; if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { if (pItem->pVar.nLen > pSchema->bytes) { @@ -6618,9 +6600,9 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } } - + ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true); - + // check again after the convert since it may be converted from binary to nchar. if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { int16_t len = varDataTLen(tagVal); @@ -6629,12 +6611,12 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } } - + if (ret != TSDB_CODE_SUCCESS) { tdDestroyKVRowBuilder(&kvRowBuilder); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); } - + tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal); } } @@ -6736,8 +6718,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { return TSDB_CODE_TSC_INVALID_SQL; } - if ((pQueryInfo->interval.interval > 0) && - (validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS)) { + if (isTimeWindowQuery(pQueryInfo) && (validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -6969,10 +6950,10 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { } int32_t joinQuery = (pQuerySql->from != NULL && taosArrayGetSize(pQuerySql->from) > 2); + int32_t timeWindowQuery = !(pQuerySql->interval.interval.type == 0 || pQuerySql->interval.interval.n == 0 || + pQuerySql->sessionVal.gap.n == 0); - int32_t intervalQuery = !(pQuerySql->interval.type == 0 || pQuerySql->interval.n == 0); - - if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable, joinQuery, intervalQuery) != TSDB_CODE_SUCCESS) { + if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable, joinQuery, timeWindowQuery) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -6985,12 +6966,16 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { if (parseIntervalClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } else { - if ((pQueryInfo->interval.interval > 0) && + if (isTimeWindowQuery(pQueryInfo) && (validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS)) { return TSDB_CODE_TSC_INVALID_SQL; } } + if (parseSessionClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_SQL; + } + // no result due to invalid query time range if (pQueryInfo->window.skey > pQueryInfo->window.ekey) { pQueryInfo->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; @@ -7004,7 +6989,6 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { // in case of join query, time range is required. if (QUERY_IS_JOIN_QUERY(pQueryInfo->type)) { int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); - if (timeRange == 0 && pQueryInfo->window.skey == 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -7018,6 +7002,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return code; } + updateLastScanOrderIfNeeded(pQueryInfo); tscFieldInfoUpdateOffset(pQueryInfo); if (pQuerySql->fillType != NULL) { @@ -7059,25 +7044,25 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS } } - if (pSqlExpr->pLeft == NULL && pSqlExpr->pRight == NULL && pSqlExpr->nSQLOptr == 0) { + if (pSqlExpr->pLeft == NULL && pSqlExpr->pRight == NULL && pSqlExpr->tokenId == 0) { *pExpr = calloc(1, sizeof(tExprNode)); return TSDB_CODE_SUCCESS; } if (pSqlExpr->pLeft == NULL) { - if (pSqlExpr->nSQLOptr >= TK_BOOL && pSqlExpr->nSQLOptr <= TK_STRING) { + if (pSqlExpr->type == SQL_NODE_VALUE) { *pExpr = calloc(1, sizeof(tExprNode)); (*pExpr)->nodeType = TSQL_NODE_VALUE; (*pExpr)->pVal = calloc(1, sizeof(tVariant)); - tVariantAssign((*pExpr)->pVal, &pSqlExpr->val); + tVariantAssign((*pExpr)->pVal, &pSqlExpr->value); return TSDB_CODE_SUCCESS; - } else if (pSqlExpr->nSQLOptr >= TK_COUNT && pSqlExpr->nSQLOptr <= TK_AVG_IRATE) { + } else if (pSqlExpr->type == SQL_NODE_SQLFUNCTION) { // arithmetic expression on the results of aggregation functions *pExpr = calloc(1, sizeof(tExprNode)); (*pExpr)->nodeType = TSQL_NODE_COL; (*pExpr)->pSchema = calloc(1, sizeof(SSchema)); - strncpy((*pExpr)->pSchema->name, pSqlExpr->operand.z, pSqlExpr->operand.n); + strncpy((*pExpr)->pSchema->name, pSqlExpr->token.z, pSqlExpr->token.n); // set the input column data byte and type. size_t size = taosArrayGetSize(pQueryInfo->exprList); @@ -7097,7 +7082,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS break; } } - } else if (pSqlExpr->nSQLOptr == TK_ID) { // column name, normal column arithmetic expression + } else if (pSqlExpr->type == SQL_NODE_TABLE_COLUMN) { // column name, normal column arithmetic expression SColumnIndex index = COLUMN_INDEX_INITIALIZER; int32_t ret = getColumnIndexByName(pCmd, &pSqlExpr->colInfo, pQueryInfo, &index); if (ret != TSDB_CODE_SUCCESS) { @@ -7137,7 +7122,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS (*pExpr)->_node.pLeft = pLeft; (*pExpr)->_node.pRight = pRight; - SStrToken t = {.type = pSqlExpr->nSQLOptr}; + SStrToken t = {.type = pSqlExpr->tokenId}; (*pExpr)->_node.optr = convertOptr(&t); assert((*pExpr)->_node.optr != 0); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index c334a4a29ca1dea422c6ae4fda472354d0ea92dc..7882d49d4a1cc63b71f413125ea4f912784cd7fb 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -497,8 +497,6 @@ int tscProcessSql(SSqlObj *pSql) { return pSql->res.code; } } else if (pCmd->command >= TSDB_SQL_LOCAL) { - //pSql->epSet = tscMgmtEpSet; -// } else { // local handler return (*tscProcessMsgRsp[pCmd->command])(pSql); } @@ -645,7 +643,6 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char } pSql->epSet.inUse = rand()%pSql->epSet.numOfEps; - pQueryMsg->head.vgId = htonl(vgId); STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; @@ -660,8 +657,6 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); assert(index >= 0 && index < numOfVgroups); - tscDebug("%p query on stable, vgIndex:%d, numOfVgroups:%d", pSql, index, numOfVgroups); - SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index); // set the vgroup info @@ -670,7 +665,10 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char int32_t numOfTables = (int32_t)taosArrayGetSize(pTableIdList->itemList); pQueryMsg->numOfTables = htonl(numOfTables); // set the number of tables - + + tscDebug("%p query on stable, vgId:%d, numOfTables:%d, vgIndex:%d, numOfVgroups:%d", pSql, + pTableIdList->vgInfo.vgId, numOfTables, index, numOfVgroups); + // serialize each table id info for(int32_t i = 0; i < numOfTables; ++i) { STableIdInfo* pItem = taosArrayGet(pTableIdList->itemList, i); @@ -705,7 +703,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList); - if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo)) { + if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo) && !tscQueryBlockInfo(pQueryInfo)) { tscError("%p illegal value of numOfCols in query msg: %" PRIu64 ", table cols:%d", pSql, (uint64_t)numOfSrcCols, tscGetNumOfColumns(pTableMeta)); @@ -756,6 +754,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->vgroupLimit = htobe64(pQueryInfo->vgroupLimit); pQueryMsg->sqlstrLen = htonl(sqlLen); pQueryMsg->prevResultLen = htonl(pQueryInfo->bufLen); + pQueryMsg->sw.gap = htobe64(pQueryInfo->sessionWindow.gap); + pQueryMsg->sw.primaryColId = htonl(PRIMARYKEY_TIMESTAMP_COL_INDEX); size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo); pQueryMsg->numOfOutput = htons((int16_t)numOfOutput); // this is the stage one output column number @@ -835,13 +835,31 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pSqlFuncExpr->colInfo.colIndex = htons(pExpr->colInfo.colIndex); pSqlFuncExpr->colInfo.flag = htons(pExpr->colInfo.flag); + if (TSDB_COL_IS_UD_COL(pExpr->colInfo.flag)) { + pSqlFuncExpr->colType = htons(pExpr->resType); + pSqlFuncExpr->colBytes = htons(pExpr->resBytes); + } else if (pExpr->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { + SSchema *s = tGetTbnameColumnSchema(); + + pSqlFuncExpr->colType = htons(s->type); + pSqlFuncExpr->colBytes = htons(s->bytes); + } else if (pExpr->colInfo.colId == TSDB_BLOCK_DIST_COLUMN_INDEX) { + SSchema s = tGetBlockDistColumnSchema(); + + pSqlFuncExpr->colType = htons(s.type); + pSqlFuncExpr->colBytes = htons(s.bytes); + } else { + SSchema* s = tscGetColumnSchemaById(pTableMeta, pExpr->colInfo.colId); + pSqlFuncExpr->colType = htons(s->type); + pSqlFuncExpr->colBytes = htons(s->bytes); + } + pSqlFuncExpr->functionId = htons(pExpr->functionId); pSqlFuncExpr->numOfParams = htons(pExpr->numOfParams); pSqlFuncExpr->resColId = htons(pExpr->resColId); pMsg += sizeof(SSqlFuncMsg); - for (int32_t j = 0; j < pExpr->numOfParams; ++j) { - // todo add log + for (int32_t j = 0; j < pExpr->numOfParams; ++j) { // todo add log pSqlFuncExpr->arg[j].argType = htons((uint16_t)pExpr->param[j].nType); pSqlFuncExpr->arg[j].argBytes = htons(pExpr->param[j].nLen); @@ -866,6 +884,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { for (int32_t i = 0; i < output; ++i) { SInternalField* pField = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, i); SSqlExpr *pExpr = pField->pSqlExpr; + + // this should be switched to projection query if (pExpr != NULL) { // the queried table has been removed and a new table with the same name has already been created already // return error msg @@ -879,33 +899,31 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { return TSDB_CODE_TSC_INVALID_SQL; } - pSqlFuncExpr1->colInfo.colId = htons(pExpr->colInfo.colId); - pSqlFuncExpr1->colInfo.colIndex = htons(pExpr->colInfo.colIndex); - pSqlFuncExpr1->colInfo.flag = htons(pExpr->colInfo.flag); - - pSqlFuncExpr1->functionId = htons(pExpr->functionId); - pSqlFuncExpr1->numOfParams = htons(pExpr->numOfParams); - pMsg += sizeof(SSqlFuncMsg); - - for (int32_t j = 0; j < pExpr->numOfParams; ++j) { - // todo add log - pSqlFuncExpr1->arg[j].argType = htons((uint16_t)pExpr->param[j].nType); - pSqlFuncExpr1->arg[j].argBytes = htons(pExpr->param[j].nLen); - - if (pExpr->param[j].nType == TSDB_DATA_TYPE_BINARY) { - memcpy(pMsg, pExpr->param[j].pz, pExpr->param[j].nLen); - pMsg += pExpr->param[j].nLen; - } else { - pSqlFuncExpr1->arg[j].argValue.i64 = htobe64(pExpr->param[j].i64); + pSqlFuncExpr1->numOfParams = 0; // no params for projection query + pSqlFuncExpr1->functionId = htons(TSDB_FUNC_PRJ); + pSqlFuncExpr1->colInfo.colId = htons(pExpr->resColId); + pSqlFuncExpr1->colInfo.flag = htons(TSDB_COL_NORMAL); + + bool assign = false; + for (int32_t f = 0; f < tscSqlExprNumOfExprs(pQueryInfo); ++f) { + SSqlExpr *pe = tscSqlExprGet(pQueryInfo, f); + if (pe == pExpr) { + pSqlFuncExpr1->colInfo.colIndex = htons(f); + pSqlFuncExpr1->colType = htons(pe->resType); + pSqlFuncExpr1->colBytes = htons(pe->resBytes); + assign = true; + break; } } + assert(assign); + pMsg += sizeof(SSqlFuncMsg); pSqlFuncExpr1 = (SSqlFuncMsg *)pMsg; } else { assert(pField->pArithExprInfo != NULL); SExprInfo* pExprInfo = pField->pArithExprInfo; - pSqlFuncExpr1->colInfo.colId = htons(pExprInfo->base.colInfo.colId); + pSqlFuncExpr1->colInfo.colId = htons(pExprInfo->base.colInfo.colId); pSqlFuncExpr1->functionId = htons(pExprInfo->base.functionId); pSqlFuncExpr1->numOfParams = htons(pExprInfo->base.numOfParams); pMsg += sizeof(SSqlFuncMsg); diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index f3d7ef28c05fc3128622f8217341d52327d79ec0..32257f5a7c723d04ec0f8200c889a68d78cf7824 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -503,9 +503,19 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) { SSqlCmd *pCmd = &pSql->cmd; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - if (taosArrayGetSize(pSub->progress) > 0) { // fix crash in single tabel subscription - pQueryInfo->window.skey = ((SSubscriptionProgress*)taosArrayGet(pSub->progress, 0))->key; - tscDebug("subscribe:%s set subscribe skey:%"PRId64, pSub->topic, pQueryInfo->window.skey); + if (taosArrayGetSize(pSub->progress) > 0) { // fix crash in single table subscription + + size_t size = taosArrayGetSize(pSub->progress); + TSKEY s = INT64_MAX; + for(int32_t i = 0; i < size; ++i) { + TSKEY k = ((SSubscriptionProgress*)taosArrayGet(pSub->progress, i))->key; + if (s > k) { + s = k; + } + } + + pQueryInfo->window.skey = s; + tscDebug("subscribe:%s set next round subscribe skey:%"PRId64, pSub->topic, pQueryInfo->window.skey); } if (pSub->pTimer == NULL) { diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 78e9c6829081cb61f5fa8e2ed55397242ba357a4..22cb5809510e0863c8aaeb71e0072a3146b31610 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -74,14 +74,14 @@ static bool allSubqueryDone(SSqlObj *pParentSql) { SSubqueryState *subState = &pParentSql->subState; //lock in caller - + tscDebug("%p total subqueries: %d", pParentSql, subState->numOfSub); for (int i = 0; i < subState->numOfSub; i++) { if (0 == subState->states[i]) { - tscDebug("%p subquery:%p,%d is NOT finished, total:%d", pParentSql, pParentSql->pSubs[i], i, subState->numOfSub); + tscDebug("%p subquery:%p, index: %d NOT finished, abort query completion check", pParentSql, pParentSql->pSubs[i], i); done = false; break; } else { - tscDebug("%p subquery:%p,%d is finished, total:%d", pParentSql, pParentSql->pSubs[i], i, subState->numOfSub); + tscDebug("%p subquery:%p, index: %d finished", pParentSql, pParentSql->pSubs[i], i); } } @@ -453,7 +453,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { pSubQueryInfo->tsBuf = NULL; // free result for async object will also free sqlObj - assert(tscSqlExprNumOfExprs(pSubQueryInfo) == 1); // ts_comp query only requires one resutl columns + assert(tscSqlExprNumOfExprs(pSubQueryInfo) == 1); // ts_comp query only requires one result columns taos_free_result(pPrevSub); SSqlObj *pNew = createSubqueryObj(pSql, (int16_t) i, tscJoinQueryCallback, pSupporter, TSDB_SQL_SELECT, NULL); @@ -507,6 +507,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, 0); int16_t funcId = pExpr->functionId; + // add the invisible timestamp column if ((pExpr->colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) || (funcId != TSDB_FUNC_TS && funcId != TSDB_FUNC_TS_DUMMY && funcId != TSDB_FUNC_PRJ)) { @@ -847,6 +848,8 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow SSqlRes* pRes = &pSql->res; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + + // todo, the type may not include TSDB_QUERY_TYPE_TAG_FILTER_QUERY assert(TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY)); if (pParentSql->res.code != TSDB_CODE_SUCCESS) { @@ -1059,7 +1062,6 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow tscError("%p invalid ts comp file from vnode, abort subquery, file size:%d", pSql, numOfRows); pParentSql->res.code = TAOS_SYSTEM_ERROR(errno); - if (quitAllSubquery(pSql, pParentSql, pSupporter)){ return; } @@ -1880,6 +1882,13 @@ void doAppendData(SInterResult* pInterResult, TAOS_ROW row, int32_t numOfCols, S } } + if (p && taosArrayGetSize(p) > 0) { + SResPair *l = taosArrayGetLast(p); + if (l->key == key && key == INT64_MIN) { + continue; + } + } + //append a new column if (p == NULL) { SStddevInterResult t = {.colId = id, .pResult = taosArrayInit(10, sizeof(SResPair)),}; @@ -1941,7 +1950,11 @@ void tscFirstRoundRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { // tag or group by column if (TSDB_COL_IS_TAG(pExpr->colInfo.flag) || pExpr->functionId == TSDB_FUNC_PRJ) { - memcpy(p + offset, row[i], length[i]); + if (row[i] == NULL) { + setNull(p + offset, pExpr->resType, pExpr->resBytes); + } else { + memcpy(p + offset, row[i], length[i]); + } offset += pExpr->resBytes; } } @@ -2639,12 +2652,17 @@ static SSqlObj *tscCreateSTableSubquery(SSqlObj *pSql, SRetrieveSupport *trsuppo SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); pQueryInfo->type |= TSDB_QUERY_TYPE_STABLE_SUBQUERY; + + // clear the limit/offset info, since it should not be sent to vnode to be executed. + pQueryInfo->limit.limit = -1; + pQueryInfo->limit.offset = 0; + assert(pQueryInfo->numOfTables == 1 && pNew->cmd.numOfClause == 1 && trsupport->subqueryIndex < pSql->subState.numOfSub); // launch subquery for each vnode, so the subquery index equals to the vgroupIndex. STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, table_index); pTableMetaInfo->vgroupIndex = trsupport->subqueryIndex; - + pSql->pSubs[trsupport->subqueryIndex] = pNew; } @@ -3098,30 +3116,6 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { } } -static UNUSED_FUNC void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pField) { - SSqlRes *pRes = &pSql->res; - - if (pRes->tsrow[columnIndex] != NULL && pField->type == TSDB_DATA_TYPE_NCHAR) { - // convert unicode to native code in a temporary buffer extra one byte for terminated symbol - if (pRes->buffer[columnIndex] == NULL) { - pRes->buffer[columnIndex] = malloc(pField->bytes + TSDB_NCHAR_SIZE); - } - - /* string terminated char for binary data*/ - memset(pRes->buffer[columnIndex], 0, pField->bytes + TSDB_NCHAR_SIZE); - - int32_t length = taosUcs4ToMbs(pRes->tsrow[columnIndex], pRes->length[columnIndex], pRes->buffer[columnIndex]); - if ( length >= 0 ) { - pRes->tsrow[columnIndex] = (unsigned char*)pRes->buffer[columnIndex]; - pRes->length[columnIndex] = length; - } else { - tscError("%p charset:%s to %s. val:%s convert failed.", pSql, DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)pRes->tsrow[columnIndex]); - pRes->tsrow[columnIndex] = NULL; - pRes->length[columnIndex] = 0; - } - } -} - char *getArithmeticInputSrc(void *param, const char *name, int32_t colId) { SArithmeticSupport *pSupport = (SArithmeticSupport *) param; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 578a7df1493b23792791a29bf0acf43c6e38b719..2d5e8382a430f144a09697b709748f605ce2625a 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -97,6 +97,22 @@ bool tscQueryTags(SQueryInfo* pQueryInfo) { return true; } +bool tscQueryBlockInfo(SQueryInfo* pQueryInfo) { + int32_t numOfCols = (int32_t) tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t i = 0; i < numOfCols; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + int32_t functId = pExpr->functionId; + + // "select count(tbname)" query + if (functId == TSDB_FUNC_BLKINFO) { + return true; + } + } + + return false; +} + bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) { if (pQueryInfo == NULL) { return false; @@ -223,6 +239,21 @@ bool tscIsSecondStageQuery(SQueryInfo* pQueryInfo) { return false; } +bool tscGroupbyColumn(SQueryInfo* pQueryInfo) { + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); + + SSqlGroupbyExpr* pGroupbyExpr = &pQueryInfo->groupbyExpr; + for (int32_t k = 0; k < pGroupbyExpr->numOfGroupCols; ++k) { + SColIndex* pIndex = taosArrayGet(pGroupbyExpr->columnInfo, k); + if (!TSDB_COL_IS_TAG(pIndex->flag) && pIndex->colIndex < numOfCols) { // group by normal columns + return true; + } + } + + return false; +} + bool tscIsTWAQuery(SQueryInfo* pQueryInfo) { size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); for (int32_t i = 0; i < numOfExprs; ++i) { @@ -1722,10 +1753,15 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo) { pQueryInfo->fieldsInfo.internalField = taosArrayInit(4, sizeof(SInternalField)); assert(pQueryInfo->exprList == NULL); - pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES); - pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); - pQueryInfo->udColumnId = TSDB_UD_COLUMN_INDEX; - pQueryInfo->resColumnId= -1000; + pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES); + pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + pQueryInfo->udColumnId = TSDB_UD_COLUMN_INDEX; + pQueryInfo->resColumnId = -1000; + pQueryInfo->limit.limit = -1; + pQueryInfo->limit.offset = 0; + + pQueryInfo->slimit.limit = -1; + pQueryInfo->slimit.offset = 0; } int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 959654d1585b13c31e96ab7972d0aa2782a51901..e8c076099759ffe683c78b55a54848cf765daada 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -283,12 +283,37 @@ typedef struct { #define keyCol(pCols) (&((pCols)->cols[0])) // Key column #define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)] #define dataColsKeyAt(pCols, idx) tdGetKey(dataColsTKeyAt(pCols, idx)) -#define dataColsTKeyFirst(pCols) (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, 0)) -#define dataColsKeyFirst(pCols) (((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, 0)) -#define dataColsTKeyLast(pCols) \ - (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, (pCols)->numOfRows - 1)) -#define dataColsKeyLast(pCols) \ - (((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, (pCols)->numOfRows - 1)) +static FORCE_INLINE TKEY dataColsTKeyFirst(SDataCols *pCols) { + if (pCols->numOfRows) { + return dataColsTKeyAt(pCols, 0); + } else { + return TKEY_INVALID; + } +} + +static FORCE_INLINE TSKEY dataColsKeyFirst(SDataCols *pCols) { + if (pCols->numOfRows) { + return dataColsKeyAt(pCols, 0); + } else { + return TSDB_DATA_TIMESTAMP_NULL; + } +} + +static FORCE_INLINE TKEY dataColsTKeyLast(SDataCols *pCols) { + if (pCols->numOfRows) { + return dataColsTKeyAt(pCols, pCols->numOfRows - 1); + } else { + return TKEY_INVALID; + } +} + +static FORCE_INLINE TSKEY dataColsKeyLast(SDataCols *pCols) { + if (pCols->numOfRows) { + return dataColsKeyAt(pCols, pCols->numOfRows - 1); + } else { + return TSDB_DATA_TIMESTAMP_NULL; + } +} SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows); void tdResetDataCols(SDataCols *pCols); diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h index b651913d7384bac75eea7c9644b501b781c47e33..465b298973cbccdf380150bdce46a09b56594801 100644 --- a/src/common/inc/tname.h +++ b/src/common/inc/tname.h @@ -33,7 +33,7 @@ typedef struct SDataStatis { typedef struct SColumnInfoData { SColumnInfo info; - void* pData; // the corresponding block data in memory + char* pData; // the corresponding block data in memory } SColumnInfoData; typedef struct SResPair { diff --git a/src/connector/python/linux/python2/setup.py b/src/connector/python/linux/python2/setup.py index 4a829f36c4bf0d6e680ed923573509cc1fad39db..d1fca047c67861706e6e6afc4bc05cd888bff755 100644 --- a/src/connector/python/linux/python2/setup.py +++ b/src/connector/python/linux/python2/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.6", + version="2.0.7", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/linux/python2/taos/cinterface.py b/src/connector/python/linux/python2/taos/cinterface.py index 555cc3435bcbea302b34cbde09772ac5f6fe32b2..4367947341edad7b0a9bdbcaec69a7de9801e267 100644 --- a/src/connector/python/linux/python2/taos/cinterface.py +++ b/src/connector/python/linux/python2/taos/cinterface.py @@ -22,10 +22,10 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): if num_of_rows > 0: return list(map(_timestamp_converter, ctypes.cast( - data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)])) else: return list(map(_timestamp_converter, ctypes.cast( - data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)])) def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): @@ -145,10 +145,10 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): """ if num_of_rows > 0: return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]] + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]] else: return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]] + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]] def _crow_bigint_unsigned_to_python( @@ -162,13 +162,13 @@ def _crow_bigint_unsigned_to_python( return [ None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( data, ctypes.POINTER( - ctypes.c_ulong))[ + ctypes.c_uint64))[ :abs(num_of_rows)]] else: return [ None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( data, ctypes.POINTER( - ctypes.c_ulong))[ + ctypes.c_uint64))[ :abs(num_of_rows)]] @@ -600,7 +600,7 @@ class CTaosInterface(object): # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_INT): # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BIGINT): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[0] + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_FLOAT): # return ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_DOUBLE): @@ -608,7 +608,7 @@ class CTaosInterface(object): # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BINARY): # return (ctypes.cast(data, ctypes.POINTER(ctypes.c_char))[0:byte]).rstrip('\x00') # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_TIMESTAMP): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[0] + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_NCHAR): # return (ctypes.cast(data, ctypes.c_char_p).value).rstrip('\x00') diff --git a/src/connector/python/linux/python3/setup.py b/src/connector/python/linux/python3/setup.py index a865f5df856d1b416d7f78da8b1f857a967f5e61..296e79b973fde7c86493565940d63f64e7c6fca3 100644 --- a/src/connector/python/linux/python3/setup.py +++ b/src/connector/python/linux/python3/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.5", + version="2.0.7", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/linux/python3/taos/cinterface.py b/src/connector/python/linux/python3/taos/cinterface.py index 555cc3435bcbea302b34cbde09772ac5f6fe32b2..4367947341edad7b0a9bdbcaec69a7de9801e267 100644 --- a/src/connector/python/linux/python3/taos/cinterface.py +++ b/src/connector/python/linux/python3/taos/cinterface.py @@ -22,10 +22,10 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): if num_of_rows > 0: return list(map(_timestamp_converter, ctypes.cast( - data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)])) else: return list(map(_timestamp_converter, ctypes.cast( - data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)])) def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): @@ -145,10 +145,10 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): """ if num_of_rows > 0: return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]] + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]] else: return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]] + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]] def _crow_bigint_unsigned_to_python( @@ -162,13 +162,13 @@ def _crow_bigint_unsigned_to_python( return [ None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( data, ctypes.POINTER( - ctypes.c_ulong))[ + ctypes.c_uint64))[ :abs(num_of_rows)]] else: return [ None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( data, ctypes.POINTER( - ctypes.c_ulong))[ + ctypes.c_uint64))[ :abs(num_of_rows)]] @@ -600,7 +600,7 @@ class CTaosInterface(object): # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_INT): # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BIGINT): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[0] + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_FLOAT): # return ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_DOUBLE): @@ -608,7 +608,7 @@ class CTaosInterface(object): # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BINARY): # return (ctypes.cast(data, ctypes.POINTER(ctypes.c_char))[0:byte]).rstrip('\x00') # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_TIMESTAMP): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[0] + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_NCHAR): # return (ctypes.cast(data, ctypes.c_char_p).value).rstrip('\x00') diff --git a/src/connector/python/linux/python3/taos/cursor.py b/src/connector/python/linux/python3/taos/cursor.py index 2e7c362d547973bc3e78f2eb57a33b7fa2d0635e..32dc0ea3c3676c61c65bf92b4fb04a5373314126 100644 --- a/src/connector/python/linux/python3/taos/cursor.py +++ b/src/connector/python/linux/python3/taos/cursor.py @@ -1,7 +1,6 @@ from .cinterface import CTaosInterface from .error import * from .constants import FieldType -import threading # querySeqNum = 0 @@ -38,7 +37,6 @@ class TDengineCursor(object): self._block_iter = 0 self._affected_rows = 0 self._logfile = "" - self._threadId = threading.get_ident() if connection is not None: self._connection = connection @@ -105,12 +103,6 @@ class TDengineCursor(object): def execute(self, operation, params=None): """Prepare and execute a database operation (query or command). """ - # if threading.get_ident() != self._threadId: - # info ="Cursor execute:Thread ID not match,creater:"+str(self._threadId)+" caller:"+str(threading.get_ident()) - # raise OperationalError(info) - # print(info) - # return None - if not operation: return None @@ -280,12 +272,6 @@ class TDengineCursor(object): def _handle_result(self): """Handle the return result from query. """ - # if threading.get_ident() != self._threadId: - # info = "Cursor handleresult:Thread ID not match,creater:"+str(self._threadId)+" caller:"+str(threading.get_ident()) - # raise OperationalError(info) - # print(info) - # return None - self._description = [] for ele in self._fields: self._description.append( diff --git a/src/connector/python/osx/python3/setup.py b/src/connector/python/osx/python3/setup.py index a6b97f753c7ee3bc303be0db2217e87e889ef4df..9bce1a976febcec457c2b3f8ade0bc6d546307c4 100644 --- a/src/connector/python/osx/python3/setup.py +++ b/src/connector/python/osx/python3/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.5", + version="2.0.7", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/osx/python3/taos/cinterface.py b/src/connector/python/osx/python3/taos/cinterface.py index 6f56cf0c5e09c14fdc9d1296c80e434ab672ef44..dca9bd42e8733616f34654542ba8c2c3ea3ece9d 100644 --- a/src/connector/python/osx/python3/taos/cinterface.py +++ b/src/connector/python/osx/python3/taos/cinterface.py @@ -22,10 +22,10 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): if num_of_rows > 0: return list(map(_timestamp_converter, ctypes.cast( - data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)])) else: return list(map(_timestamp_converter, ctypes.cast( - data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)])) def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): @@ -145,10 +145,10 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): """ if num_of_rows > 0: return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]] + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]] else: return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]] + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]] def _crow_bigint_unsigned_to_python( @@ -162,13 +162,13 @@ def _crow_bigint_unsigned_to_python( return [ None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( data, ctypes.POINTER( - ctypes.c_ulong))[ + ctypes.c_uint64))[ :abs(num_of_rows)]] else: return [ None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( data, ctypes.POINTER( - ctypes.c_ulong))[ + ctypes.c_uint64))[ :abs(num_of_rows)]] @@ -600,7 +600,7 @@ class CTaosInterface(object): # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_INT): # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BIGINT): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[0] + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_FLOAT): # return ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_DOUBLE): @@ -608,7 +608,7 @@ class CTaosInterface(object): # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BINARY): # return (ctypes.cast(data, ctypes.POINTER(ctypes.c_char))[0:byte]).rstrip('\x00') # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_TIMESTAMP): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[0] + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_NCHAR): # return (ctypes.cast(data, ctypes.c_char_p).value).rstrip('\x00') diff --git a/src/connector/python/osx/python3/taos/cursor.py b/src/connector/python/osx/python3/taos/cursor.py index 2e7c362d547973bc3e78f2eb57a33b7fa2d0635e..32dc0ea3c3676c61c65bf92b4fb04a5373314126 100644 --- a/src/connector/python/osx/python3/taos/cursor.py +++ b/src/connector/python/osx/python3/taos/cursor.py @@ -1,7 +1,6 @@ from .cinterface import CTaosInterface from .error import * from .constants import FieldType -import threading # querySeqNum = 0 @@ -38,7 +37,6 @@ class TDengineCursor(object): self._block_iter = 0 self._affected_rows = 0 self._logfile = "" - self._threadId = threading.get_ident() if connection is not None: self._connection = connection @@ -105,12 +103,6 @@ class TDengineCursor(object): def execute(self, operation, params=None): """Prepare and execute a database operation (query or command). """ - # if threading.get_ident() != self._threadId: - # info ="Cursor execute:Thread ID not match,creater:"+str(self._threadId)+" caller:"+str(threading.get_ident()) - # raise OperationalError(info) - # print(info) - # return None - if not operation: return None @@ -280,12 +272,6 @@ class TDengineCursor(object): def _handle_result(self): """Handle the return result from query. """ - # if threading.get_ident() != self._threadId: - # info = "Cursor handleresult:Thread ID not match,creater:"+str(self._threadId)+" caller:"+str(threading.get_ident()) - # raise OperationalError(info) - # print(info) - # return None - self._description = [] for ele in self._fields: self._description.append( diff --git a/src/connector/python/windows/python2/setup.py b/src/connector/python/windows/python2/setup.py index 333f5bedad7edeefab66e1c1ba90424085e2aa1c..47d374fe67673172596824f68ed495f68941bb51 100644 --- a/src/connector/python/windows/python2/setup.py +++ b/src/connector/python/windows/python2/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.4", + version="2.0.7", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/windows/python2/taos/cinterface.py b/src/connector/python/windows/python2/taos/cinterface.py index d8cdce2ad138c34db5193e3972ba51d46c693254..ec72474df93a4d13aa13256611044c5bc140f2d3 100644 --- a/src/connector/python/windows/python2/taos/cinterface.py +++ b/src/connector/python/windows/python2/taos/cinterface.py @@ -22,10 +22,10 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): if num_of_rows > 0: return list(map(_timestamp_converter, ctypes.cast( - data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)])) else: return list(map(_timestamp_converter, ctypes.cast( - data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)])) def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): @@ -145,10 +145,10 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): """ if num_of_rows > 0: return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]] + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]] else: return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]] + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]] def _crow_bigint_unsigned_to_python( @@ -162,13 +162,13 @@ def _crow_bigint_unsigned_to_python( return [ None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( data, ctypes.POINTER( - ctypes.c_ulong))[ + ctypes.c_uint64))[ :abs(num_of_rows)]] else: return [ None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( data, ctypes.POINTER( - ctypes.c_ulong))[ + ctypes.c_uint64))[ :abs(num_of_rows)]] @@ -600,7 +600,7 @@ class CTaosInterface(object): # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_INT): # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BIGINT): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[0] + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_FLOAT): # return ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_DOUBLE): @@ -608,7 +608,7 @@ class CTaosInterface(object): # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BINARY): # return (ctypes.cast(data, ctypes.POINTER(ctypes.c_char))[0:byte]).rstrip('\x00') # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_TIMESTAMP): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[0] + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_NCHAR): # return (ctypes.cast(data, ctypes.c_char_p).value).rstrip('\x00') diff --git a/src/connector/python/windows/python2/taos/cursor.py b/src/connector/python/windows/python2/taos/cursor.py index 0656b6326e173b111eb8293c6e3b76678eccc0e2..5f4666b59396d4dbd7df6daaf62beb814e3b53f7 100644 --- a/src/connector/python/windows/python2/taos/cursor.py +++ b/src/connector/python/windows/python2/taos/cursor.py @@ -1,7 +1,6 @@ from .cinterface import CTaosInterface from .error import * from .constants import FieldType -import threading # querySeqNum = 0 @@ -38,7 +37,6 @@ class TDengineCursor(object): self._block_iter = 0 self._affected_rows = 0 self._logfile = "" - self._threadId = threading.get_ident() if connection is not None: self._connection = connection diff --git a/src/connector/python/windows/python3/setup.py b/src/connector/python/windows/python3/setup.py index f29fec121b27f5e87ffd215bb4ff5006acf3bf1c..cdcec62a218e72adbda949c0fd666b5ec4abc340 100644 --- a/src/connector/python/windows/python3/setup.py +++ b/src/connector/python/windows/python3/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.4", + version="2.0.7", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/windows/python3/taos/cinterface.py b/src/connector/python/windows/python3/taos/cinterface.py index d8cdce2ad138c34db5193e3972ba51d46c693254..ec72474df93a4d13aa13256611044c5bc140f2d3 100644 --- a/src/connector/python/windows/python3/taos/cinterface.py +++ b/src/connector/python/windows/python3/taos/cinterface.py @@ -22,10 +22,10 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): if num_of_rows > 0: return list(map(_timestamp_converter, ctypes.cast( - data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)])) else: return list(map(_timestamp_converter, ctypes.cast( - data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)])) def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): @@ -145,10 +145,10 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): """ if num_of_rows > 0: return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]] + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]] else: return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( - data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]] + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]] def _crow_bigint_unsigned_to_python( @@ -162,13 +162,13 @@ def _crow_bigint_unsigned_to_python( return [ None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( data, ctypes.POINTER( - ctypes.c_ulong))[ + ctypes.c_uint64))[ :abs(num_of_rows)]] else: return [ None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( data, ctypes.POINTER( - ctypes.c_ulong))[ + ctypes.c_uint64))[ :abs(num_of_rows)]] @@ -600,7 +600,7 @@ class CTaosInterface(object): # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_INT): # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BIGINT): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[0] + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_FLOAT): # return ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_DOUBLE): @@ -608,7 +608,7 @@ class CTaosInterface(object): # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BINARY): # return (ctypes.cast(data, ctypes.POINTER(ctypes.c_char))[0:byte]).rstrip('\x00') # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_TIMESTAMP): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[0] + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_NCHAR): # return (ctypes.cast(data, ctypes.c_char_p).value).rstrip('\x00') diff --git a/src/connector/python/windows/python3/taos/cursor.py b/src/connector/python/windows/python3/taos/cursor.py index 769cb7cf0f61fe850c16315bf552162f33536502..136cd42fe47a51a34b301730745d73e893861f7c 100644 --- a/src/connector/python/windows/python3/taos/cursor.py +++ b/src/connector/python/windows/python3/taos/cursor.py @@ -1,7 +1,6 @@ from .cinterface import CTaosInterface from .error import * from .constants import FieldType -import threading # querySeqNum = 0 @@ -38,7 +37,6 @@ class TDengineCursor(object): self._block_iter = 0 self._affected_rows = 0 self._logfile = "" - self._threadId = threading.get_ident() if connection is not None: self._connection = connection diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 9b7b746116619464cd49e03ef13bc1267f1ec502..20984b5c7f9cbc002edaf3279885aff3a7181b75 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -244,6 +244,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO TAOS_DEF_ERROR_CODE(0, 0x0612) //"Invalid information to create table") #define TSDB_CODE_TDB_NO_AVAIL_DISK TAOS_DEF_ERROR_CODE(0, 0x0613) //"No available disk") #define TSDB_CODE_TDB_MESSED_MSG TAOS_DEF_ERROR_CODE(0, 0x0614) //"TSDB messed message") +#define TSDB_CODE_TDB_IVLD_TAG_VAL TAOS_DEF_ERROR_CODE(0, 0x0615) //"TSDB invalid tag value") // query #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) //"Invalid handle") @@ -258,7 +259,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_QRY_IN_EXEC TAOS_DEF_ERROR_CODE(0, 0x0709) //"Multiple retrieval of this query") #define TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW TAOS_DEF_ERROR_CODE(0, 0x070A) //"Too many time window in query") #define TSDB_CODE_QRY_NOT_ENOUGH_BUFFER TAOS_DEF_ERROR_CODE(0, 0x070B) //"Query buffer limit has reached") -#define TSDB_CODE_QRY_INCONSISTAN TAOS_DEF_ERROR_CODE(0, 0x070C) //"File inconsistance in replica") +#define TSDB_CODE_QRY_INCONSISTAN TAOS_DEF_ERROR_CODE(0, 0x070C) //"File inconsistency in replica") // grant diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 3518a8104c4771de8f85f060adf7afb4699c3728..990b7c6af30740b75013fb214428151549561139 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -397,7 +397,7 @@ typedef struct SColIndex { int16_t colId; // column id int16_t colIndex; // column index in colList if it is a normal column or index in tagColList if a tag uint16_t flag; // denote if it is a tag or a normal column - char name[TSDB_COL_NAME_LEN]; + char name[TSDB_COL_NAME_LEN]; // TODO remove it } SColIndex; /* sql function msg, to describe the message to vnode about sql function @@ -405,7 +405,10 @@ typedef struct SColIndex { typedef struct SSqlFuncMsg { int16_t functionId; int16_t numOfParams; + int16_t resColId; // result column id, id of the current output column + int16_t colType; + int16_t colBytes; SColIndex colInfo; struct ArgElem { @@ -485,12 +488,13 @@ typedef struct { int16_t orderColId; int16_t numOfCols; // the number of columns will be load from vnode SInterval interval; + SSessionWindow sw; // session window uint16_t tagCondLen; // tag length in current query uint32_t tbnameCondLen; // table name filter condition string length int16_t numOfGroupCols; // num of group by columns int16_t orderByIdx; int16_t orderType; // used in group by xx order by xxx - int64_t vgroupLimit; // limit the number of rows for each table, used in order by + limit in stable projection query. + int64_t vgroupLimit; // limit the number of rows for each table, used in order by + limit in stable projection query. int16_t prjOrder; // global order in super table projection query. int64_t limit; int64_t offset; @@ -640,6 +644,7 @@ typedef struct { int32_t maxtablesPerVnode; int32_t maxVgroupsPerDb; char arbitrator[TSDB_EP_LEN]; // tsArbitrator + char reserve[2]; // to solve arm32 bus error char timezone[64]; // tsTimezone int64_t checkTime; // 1970-01-01 00:00:00.000 char locale[TSDB_LOCALE_LEN]; // tsLocale diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 78cd2927c7b85d39dd0a965971334e4c3b1cbddd..09444bb8e4b2c6cd45ce087a3444338d140a90ae 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -158,13 +158,18 @@ int32_t tsdbInsertData(STsdbRepo *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pR typedef void *TsdbQueryHandleT; // Use void to hide implementation details -// query condition to build vnode iterator +#define BLOCK_LOAD_OFFSET_SEQ_ORDER 1 +#define BLOCK_LOAD_TABLE_SEQ_ORDER 2 +#define BLOCK_LOAD_TABLE_RR_ORDER 3 + +// query condition to build multi-table data block iterator typedef struct STsdbQueryCond { STimeWindow twindow; int32_t order; // desc|asc order to iterate the data block int32_t numOfCols; SColumnInfo *colList; bool loadExternalRows; // load external rows or not + int32_t type; // data block load type: } STsdbQueryCond; typedef struct SMemRef { @@ -181,17 +186,31 @@ typedef struct SDataBlockInfo { int32_t tid; } SDataBlockInfo; +typedef struct SFileBlockInfo { + int32_t numOfRows; +} SFileBlockInfo; + typedef struct { void *pTable; TSKEY lastKey; } STableKeyInfo; typedef struct { - size_t numOfTables; + uint32_t numOfTables; SArray * pGroupList; SHashObj *map; // speedup acquire the tableQueryInfo by table uid } STableGroupInfo; +typedef struct { + uint16_t rowSize; + uint16_t numOfFiles; + uint32_t numOfTables; + uint64_t totalSize; + int32_t firstSeekTimeUs; + uint32_t numOfRowsInMemTable; + SArray *dataBlockInfos; +} STableBlockDist; + /** * Get the data block iterator, starting from position according to the query condition * @@ -252,16 +271,7 @@ int64_t tsdbGetNumOfRowsInMemTable(TsdbQueryHandleT* pHandle); * @param pQueryHandle * @return */ -bool tsdbNextDataBlock(TsdbQueryHandleT *pQueryHandle); -/** - * move to next block if exists but not merge data in memtable - * - * @param pQueryHandle - * @return - */ -bool tsdbNextDataBlockWithoutMerge(TsdbQueryHandleT *pQueryHandle); - -SArray* tsdbGetExternalRow(TsdbQueryHandleT *pHandle, SMemRef* pMemRef, int16_t type); +bool tsdbNextDataBlock(TsdbQueryHandleT pQueryHandle); /** * Get current data block information @@ -306,7 +316,7 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo *tsdb, uint64_t uid, TSKEY key, const SColIndex *pColIndex, int32_t numOfCols); /** - * destory the created table group list, which is generated by tag query + * destroy the created table group list, which is generated by tag query * @param pGroupList */ void tsdbDestroyTableGroup(STableGroupInfo *pGroupList); @@ -336,6 +346,12 @@ int32_t tsdbGetTableGroupFromIdList(STsdbRepo *tsdb, SArray *pTableIdList, STabl */ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle); +void tsdbResetQueryHandle(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond); + +void tsdbResetQueryHandleForNewTable(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond, STableGroupInfo* groupList); + +int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist* pTableBlockInfo); + /** * get the statistics of repo usage * @param repo. point to the tsdbrepo diff --git a/src/inc/ttokendef.h b/src/inc/ttokendef.h index 3130894b30a8a29a226d4d83f42e8fc6092c7b01..b148d6bc759bfd727853728d0ec41aea8448a86e 100644 --- a/src/inc/ttokendef.h +++ b/src/inc/ttokendef.h @@ -138,100 +138,76 @@ #define TK_FROM 119 #define TK_VARIABLE 120 #define TK_INTERVAL 121 -#define TK_FILL 122 -#define TK_SLIDING 123 -#define TK_ORDER 124 -#define TK_BY 125 -#define TK_ASC 126 -#define TK_DESC 127 -#define TK_GROUP 128 -#define TK_HAVING 129 -#define TK_LIMIT 130 -#define TK_OFFSET 131 -#define TK_SLIMIT 132 -#define TK_SOFFSET 133 -#define TK_WHERE 134 -#define TK_NOW 135 -#define TK_RESET 136 -#define TK_QUERY 137 -#define TK_ADD 138 -#define TK_COLUMN 139 -#define TK_TAG 140 -#define TK_CHANGE 141 -#define TK_SET 142 -#define TK_KILL 143 -#define TK_CONNECTION 144 -#define TK_STREAM 145 -#define TK_COLON 146 -#define TK_ABORT 147 -#define TK_AFTER 148 -#define TK_ATTACH 149 -#define TK_BEFORE 150 -#define TK_BEGIN 151 -#define TK_CASCADE 152 -#define TK_CLUSTER 153 -#define TK_CONFLICT 154 -#define TK_COPY 155 -#define TK_DEFERRED 156 -#define TK_DELIMITERS 157 -#define TK_DETACH 158 -#define TK_EACH 159 -#define TK_END 160 -#define TK_EXPLAIN 161 -#define TK_FAIL 162 -#define TK_FOR 163 -#define TK_IGNORE 164 -#define TK_IMMEDIATE 165 -#define TK_INITIALLY 166 -#define TK_INSTEAD 167 -#define TK_MATCH 168 -#define TK_KEY 169 -#define TK_OF 170 -#define TK_RAISE 171 -#define TK_REPLACE 172 -#define TK_RESTRICT 173 -#define TK_ROW 174 -#define TK_STATEMENT 175 -#define TK_TRIGGER 176 -#define TK_VIEW 177 -#define TK_COUNT 178 -#define TK_SUM 179 -#define TK_AVG 180 -#define TK_MIN 181 -#define TK_MAX 182 -#define TK_FIRST 183 -#define TK_LAST 184 -#define TK_TOP 185 -#define TK_BOTTOM 186 -#define TK_STDDEV 187 -#define TK_PERCENTILE 188 -#define TK_APERCENTILE 189 -#define TK_LEASTSQUARES 190 -#define TK_HISTOGRAM 191 -#define TK_DIFF 192 -#define TK_SPREAD 193 -#define TK_TWA 194 -#define TK_INTERP 195 -#define TK_LAST_ROW 196 -#define TK_RATE 197 -#define TK_IRATE 198 -#define TK_SUM_RATE 199 -#define TK_SUM_IRATE 200 -#define TK_AVG_RATE 201 -#define TK_AVG_IRATE 202 -#define TK_TBID 203 -#define TK_SEMI 204 -#define TK_NONE 205 -#define TK_PREV 206 -#define TK_LINEAR 207 -#define TK_IMPORT 208 -#define TK_METRIC 209 -#define TK_TBNAME 210 -#define TK_JOIN 211 -#define TK_METRICS 212 -#define TK_INSERT 213 -#define TK_INTO 214 -#define TK_VALUES 215 +#define TK_SESSION 122 +#define TK_FILL 123 +#define TK_SLIDING 124 +#define TK_ORDER 125 +#define TK_BY 126 +#define TK_ASC 127 +#define TK_DESC 128 +#define TK_GROUP 129 +#define TK_HAVING 130 +#define TK_LIMIT 131 +#define TK_OFFSET 132 +#define TK_SLIMIT 133 +#define TK_SOFFSET 134 +#define TK_WHERE 135 +#define TK_NOW 136 +#define TK_RESET 137 +#define TK_QUERY 138 +#define TK_ADD 139 +#define TK_COLUMN 140 +#define TK_TAG 141 +#define TK_CHANGE 142 +#define TK_SET 143 +#define TK_KILL 144 +#define TK_CONNECTION 145 +#define TK_STREAM 146 +#define TK_COLON 147 +#define TK_ABORT 148 +#define TK_AFTER 149 +#define TK_ATTACH 150 +#define TK_BEFORE 151 +#define TK_BEGIN 152 +#define TK_CASCADE 153 +#define TK_CLUSTER 154 +#define TK_CONFLICT 155 +#define TK_COPY 156 +#define TK_DEFERRED 157 +#define TK_DELIMITERS 158 +#define TK_DETACH 159 +#define TK_EACH 160 +#define TK_END 161 +#define TK_EXPLAIN 162 +#define TK_FAIL 163 +#define TK_FOR 164 +#define TK_IGNORE 165 +#define TK_IMMEDIATE 166 +#define TK_INITIALLY 167 +#define TK_INSTEAD 168 +#define TK_MATCH 169 +#define TK_KEY 170 +#define TK_OF 171 +#define TK_RAISE 172 +#define TK_REPLACE 173 +#define TK_RESTRICT 174 +#define TK_ROW 175 +#define TK_STATEMENT 176 +#define TK_TRIGGER 177 +#define TK_VIEW 178 +#define TK_SEMI 179 +#define TK_NONE 180 +#define TK_PREV 181 +#define TK_LINEAR 182 +#define TK_IMPORT 183 +#define TK_METRIC 184 +#define TK_TBNAME 185 +#define TK_JOIN 186 +#define TK_METRICS 187 +#define TK_INSERT 188 +#define TK_INTO 189 +#define TK_VALUES 190 + diff --git a/src/inc/ttype.h b/src/inc/ttype.h index 2f6b80f89d024e3d286ae20b41f645a073ddc174..662a23bfdbc52e432d2bcfbdac5c43cfba5d60dc 100644 --- a/src/inc/ttype.h +++ b/src/inc/ttype.h @@ -171,10 +171,10 @@ extern tDataTypeDescriptor tDataTypes[15]; bool isValidDataType(int32_t type); -void setVardataNull(char* val, int32_t type); -void setNull(char *val, int32_t type, int32_t bytes); -void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems); -void* getNullValue(int32_t type); +void setVardataNull(char* val, int32_t type); +void setNull(char *val, int32_t type, int32_t bytes); +void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems); +void *getNullValue(int32_t type); void assignVal(char *val, const char *src, int32_t len, int32_t type); void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf); diff --git a/src/kit/taosdemo/insert.json b/src/kit/taosdemo/insert.json index 5276b9fb61af9645578392a0035c8a95898c75e2..5ed744cde9b337e2a81d43c7c2022a6b180d0ce6 100644 --- a/src/kit/taosdemo/insert.json +++ b/src/kit/taosdemo/insert.json @@ -11,6 +11,7 @@ "confirm_parameter_prompt": "no", "insert_interval": 0, "num_of_records_per_req": 100, + "max_sql_len": 1024000, "databases": [{ "dbinfo": { "name": "db", @@ -38,7 +39,9 @@ "auto_create_table": "no", "data_source": "rand", "insert_mode": "taosc", - "insert_rows": 100000, + "childtable_limit": 33, + "childtable_offset": 33, + "insert_rows": 1000, "multi_thread_write_one_tbl": "no", "number_of_tbl_in_one_sql": 0, "rows_per_tbl": 100, diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index c585e91c50e8d168f9df503cc8f1b44cf24648c8..f561c1d610afc1cfea149c35591ea0f65b5f0843 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -22,7 +22,6 @@ #define CURL_STATICLIB #ifdef LINUX - #include "os.h" #include #include #include @@ -33,7 +32,6 @@ #include #include #include - #include #include #include #include @@ -44,18 +42,12 @@ #include #include #include - #include "os.h" - -#ifdef WINDOWS - #include - typedef unsigned __int32 uint32_t; - - #pragma comment ( lib, "ws2_32.lib" ) -#endif #endif +#include #include "cJSON.h" +#include "os.h" #include "taos.h" #include "taoserror.h" #include "tutil.h" @@ -98,6 +90,8 @@ extern char configDir[]; #define MAX_DATABASE_COUNT 256 #define INPUT_BUF_LEN 256 +#define DEFAULT_TIMESTAMP_STEP 10 + typedef enum CREATE_SUB_TALBE_MOD_EN { PRE_CREATE_SUBTBL, AUTO_CREATE_SUBTBL, @@ -196,6 +190,7 @@ typedef struct SArguments_S { int num_of_threads; int insert_interval; int num_of_RPR; + int max_sql_len; int num_of_tables; int num_of_DPT; int abort; @@ -222,6 +217,8 @@ typedef struct SSuperTable_S { char childTblPrefix[MAX_TB_NAME_SIZE]; char dataSource[MAX_TB_NAME_SIZE+1]; // rand_gen or sample char insertMode[MAX_TB_NAME_SIZE]; // taosc, restful + int childTblLimit; + int childTblOffset; int multiThreadWriteOneTbl; // 0: no, 1: yes int numberOfTblInOneSql; // 0/1: one table, > 1: number of tbl @@ -229,7 +226,8 @@ typedef struct SSuperTable_S { int disorderRatio; // 0: no disorder, >0: x% int disorderRange; // ms or us by database precision int maxSqlLen; // - + + int insertInterval; // insert interval, will override global insert interval int64_t insertRows; // 0: no limit int timeStampStep; char startTimestamp[MAX_TB_NAME_SIZE]; // @@ -258,7 +256,7 @@ typedef struct SSuperTable_S { int tagUsePos; // statistics - int64_t totalRowsInserted; + int64_t totalInsertRows; int64_t totalAffectedRows; } SSuperTable; @@ -328,7 +326,7 @@ typedef struct SDbs_S { SDataBase db[MAX_DB_COUNT]; // statistics - int64_t totalRowsInserted; + int64_t totalInsertRows; int64_t totalAffectedRows; } SDbs; @@ -399,7 +397,7 @@ typedef struct SThreadInfo_S { int64_t lastTs; // statistics - int64_t totalRowsInserted; + int64_t totalInsertRows; int64_t totalAffectedRows; // insert delay statistics @@ -412,11 +410,18 @@ typedef struct SThreadInfo_S { } threadInfo; #ifdef WINDOWS +#define _CRT_RAND_S + #include +#include + +typedef unsigned __int32 uint32_t; + +#pragma comment ( lib, "ws2_32.lib" ) // Some old MinGW/CYGWIN distributions don't define this: #ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING -#define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004 -#endif + #define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004 +#endif // ENABLE_VIRTUAL_TERMINAL_PROCESSING static HANDLE g_stdoutHandle; static DWORD g_consoleMode; @@ -452,6 +457,14 @@ void resetAfterAnsiEscape(void) { exit(GetLastError()); } } + +int taosRandom() +{ + int number; + rand_s(&number); + + return number; +} #else void setupForAnsiEscape(void) {} @@ -459,6 +472,12 @@ void resetAfterAnsiEscape(void) { // Reset colors printf("\x1b[0m"); } + +int taosRandom() +{ + return random(); +} + #endif static int createDatabases(); @@ -475,8 +494,8 @@ char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)", "max(col0)", "min(col0)", "first(col0)", "last(col0)"}; SArguments g_args = { - NULL, // metaFile - 0, // test_mode + NULL, // metaFile + 0, // test_mode "127.0.0.1", // host 6030, // port "root", // user @@ -513,6 +532,7 @@ SArguments g_args = { 10, // num_of_connections/thread 0, // insert_interval 100, // num_of_RPR + TSDB_PAYLOAD_SIZE, // max_sql_len 10000, // num_of_tables 10000, // num_of_DPT 0, // abort @@ -739,7 +759,8 @@ void parse_args(int argc, char *argv[], SArguments *arguments) { } } - if (arguments->debug_print) { + if (((arguments->debug_print) && (arguments->metaFile == NULL)) + || arguments->verbose_print) { printf("###################################################################\n"); printf("# meta file: %s\n", arguments->metaFile); printf("# Server IP: %s:%hu\n", @@ -759,6 +780,8 @@ void parse_args(int argc, char *argv[], SArguments *arguments) { } printf("# Insertion interval: %d\n", arguments->insert_interval); printf("# Number of records per req: %d\n", arguments->num_of_RPR); + printf("# Max SQL length: %d\n", arguments->max_sql_len); + printf("# Length of Binary: %d\n", arguments->len_of_binary); printf("# Number of Threads: %d\n", arguments->num_of_threads); printf("# Number of Tables: %d\n", arguments->num_of_tables); printf("# Number of Data per Table: %d\n", arguments->num_of_DPT); @@ -772,6 +795,7 @@ void parse_args(int argc, char *argv[], SArguments *arguments) { printf("# Delete method: %d\n", arguments->method_of_delete); printf("# Answer yes when prompt: %d\n", arguments->answer_yes); printf("# Print debug info: %d\n", arguments->debug_print); + printf("# Print verbose info: %d\n", arguments->verbose_print); printf("###################################################################\n"); if (!arguments->answer_yes) { printf("Press enter key to continue\n\n"); @@ -966,10 +990,10 @@ static double rand_double() { static void init_rand_data() { for (int i = 0; i < MAX_PREPARED_RAND; i++){ - randint[i] = (int)(rand() % 65535); - randbigint[i] = (int64_t)(rand() % 2147483648); - randfloat[i] = (float)(rand() / 1000.0); - randdouble[i] = (double)(rand() / 1000000.0); + randint[i] = (int)(taosRandom() % 65535); + randbigint[i] = (int64_t)(taosRandom() % 2147483648); + randfloat[i] = (float)(taosRandom() / 1000.0); + randdouble[i] = (double)(taosRandom() / 1000000.0); } } @@ -1004,6 +1028,7 @@ static int printfInsertMeta() { printf("thread num of create table: \033[33m%d\033[0m\n", g_Dbs.threadCountByCreateTbl); printf("insert interval: \033[33m%d\033[0m\n", g_args.insert_interval); printf("number of records per req: \033[33m%d\033[0m\n", g_args.num_of_RPR); + printf("max sql length: \033[33m%d\033[0m\n", g_args.max_sql_len); printf("database count: \033[33m%d\033[0m\n", g_Dbs.dbCount); for (int i = 0; i < g_Dbs.dbCount; i++) { @@ -1083,37 +1108,55 @@ static int printfInsertMeta() { } else { printf(" childTblExists: \033[33m%s\033[0m\n", "error"); } - - printf(" childTblCount: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].childTblCount); - printf(" childTblPrefix: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].childTblPrefix); - printf(" dataSource: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].dataSource); - printf(" insertMode: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].insertMode); - printf(" insertRows: \033[33m%"PRId64"\033[0m\n", g_Dbs.db[i].superTbls[j].insertRows); + + printf(" childTblCount: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].childTblCount); + printf(" childTblPrefix: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].childTblPrefix); + printf(" dataSource: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].dataSource); + printf(" insertMode: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].insertMode); + if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) { + printf(" childTblLimit: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].childTblLimit); + } + if (g_Dbs.db[i].superTbls[j].childTblOffset > 0) { + printf(" childTblOffset: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].childTblOffset); + } + printf(" insertRows: \033[33m%"PRId64"\033[0m\n", g_Dbs.db[i].superTbls[j].insertRows); if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { - printf(" multiThreadWriteOneTbl: \033[33mno\033[0m\n"); + printf(" multiThreadWriteOneTbl: \033[33mno\033[0m\n"); }else { - printf(" multiThreadWriteOneTbl: \033[33myes\033[0m\n"); + printf(" multiThreadWriteOneTbl: \033[33myes\033[0m\n"); } - printf(" numberOfTblInOneSql: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].numberOfTblInOneSql); - printf(" rowsPerTbl: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].rowsPerTbl); - printf(" disorderRange: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].disorderRange); - printf(" disorderRatio: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].disorderRatio); - printf(" maxSqlLen: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].maxSqlLen); - - printf(" timeStampStep: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].timeStampStep); - printf(" startTimestamp: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].startTimestamp); - printf(" sampleFormat: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].sampleFormat); - printf(" sampleFile: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].sampleFile); - printf(" tagsFile: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].tagsFile); - - printf(" columnCount: \033[33m%d\033[0m\n ", g_Dbs.db[i].superTbls[j].columnCount); + printf(" numberOfTblInOneSql: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTbls[j].numberOfTblInOneSql); + printf(" rowsPerTbl: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTbls[j].rowsPerTbl); + printf(" disorderRange: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTbls[j].disorderRange); + printf(" disorderRatio: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTbls[j].disorderRatio); + printf(" maxSqlLen: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTbls[j].maxSqlLen); + printf(" timeStampStep: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTbls[j].timeStampStep); + printf(" startTimestamp: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].startTimestamp); + printf(" sampleFormat: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].sampleFormat); + printf(" sampleFile: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].sampleFile); + printf(" tagsFile: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].tagsFile); + printf(" columnCount: \033[33m%d\033[0m\n ", + g_Dbs.db[i].superTbls[j].columnCount); for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) { //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); - if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, "binary", 6)) - || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, "nchar", 5))) { + if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, + "binary", 6)) + || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, + "nchar", 5))) { printf("column[\033[33m%d\033[0m]:\033[33m%s(%d)\033[0m ", k, - g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); + g_Dbs.db[i].superTbls[j].columns[k].dataType, + g_Dbs.db[i].superTbls[j].columns[k].dataLen); } else { printf("column[%d]:\033[33m%s\033[0m ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType); @@ -1128,11 +1171,12 @@ static int printfInsertMeta() { if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "binary", 6)) || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "nchar", 5))) { printf("tag[%d]:\033[33m%s(%d)\033[0m ", k, - g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); + g_Dbs.db[i].superTbls[j].tags[k].dataType, + g_Dbs.db[i].superTbls[j].tags[k].dataLen); } else { printf("tag[%d]:\033[33m%s\033[0m ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType); - } + } } printf("\n"); } @@ -1236,6 +1280,7 @@ static void printfInsertMetaToFile(FILE* fp) { fprintf(fp, " dataSource: %s\n", g_Dbs.db[i].superTbls[j].dataSource); fprintf(fp, " insertMode: %s\n", g_Dbs.db[i].superTbls[j].insertMode); fprintf(fp, " insertRows: %"PRId64"\n", g_Dbs.db[i].superTbls[j].insertRows); + fprintf(fp, " insert interval: %d\n", g_Dbs.db[i].superTbls[j].insertInterval); if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { fprintf(fp, " multiThreadWriteOneTbl: no\n"); @@ -1765,21 +1810,21 @@ int postProceSql(char* host, uint16_t port, char* sqlstr) return 0; } - -char* getTagValueFromTagSample( SSuperTable* stbInfo, int tagUsePos) { +static char* getTagValueFromTagSample(SSuperTable* stbInfo, int tagUsePos) { char* dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1); if (NULL == dataBuf) { printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1); return NULL; } - + int dataLen = 0; - dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "(%s)", stbInfo->tagDataBuf + stbInfo->lenOfTagOfOneRow * tagUsePos); - + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, + "(%s)", stbInfo->tagDataBuf + stbInfo->lenOfTagOfOneRow * tagUsePos); + return dataBuf; } -char* generateTagVaulesForStb(SSuperTable* stbInfo) { +static char* generateTagVaulesForStb(SSuperTable* stbInfo) { char* dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1); if (NULL == dataBuf) { printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1); @@ -1789,13 +1834,15 @@ char* generateTagVaulesForStb(SSuperTable* stbInfo) { int dataLen = 0; dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "("); for (int i = 0; i < stbInfo->tagCount; i++) { - if ((0 == strncasecmp(stbInfo->tags[i].dataType, "binary", 6)) || (0 == strncasecmp(stbInfo->tags[i].dataType, "nchar", 5))) { + if ((0 == strncasecmp(stbInfo->tags[i].dataType, "binary", strlen("binary"))) + || (0 == strncasecmp(stbInfo->tags[i].dataType, "nchar", strlen("nchar")))) { if (stbInfo->tags[i].dataLen > TSDB_MAX_BINARY_LEN) { - printf("binary or nchar length overflow, max size:%u\n", (uint32_t)TSDB_MAX_BINARY_LEN); + printf("binary or nchar length overflow, max size:%u\n", + (uint32_t)TSDB_MAX_BINARY_LEN); tmfree(dataBuf); return NULL; } - + char* buf = (char*)calloc(stbInfo->tags[i].dataLen+1, 1); if (NULL == buf) { printf("calloc failed! size:%d\n", stbInfo->tags[i].dataLen); @@ -1803,30 +1850,48 @@ char* generateTagVaulesForStb(SSuperTable* stbInfo) { return NULL; } rand_string(buf, stbInfo->tags[i].dataLen); - dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "\'%s\', ", buf); + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, + "\'%s\', ", buf); tmfree(buf); - } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "int", 3)) { - dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_int()); - } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "bigint", 6)) { - dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%"PRId64", ", rand_bigint()); - } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "float", 5)) { - dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%f, ", rand_float()); - } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "double", 6)) { - dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%f, ", rand_double()); - } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "smallint", 8)) { - dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_smallint()); - } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "tinyint", 7)) { - dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_tinyint()); - } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "bool", 4)) { - dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_bool()); - } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "timestamp", 4)) { - dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%"PRId64", ", rand_bigint()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, + "int", strlen("int"))) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, + "%d, ", rand_int()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, + "bigint", strlen("bigint"))) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, + "%"PRId64", ", rand_bigint()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, + "float", strlen("float"))) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, + "%f, ", rand_float()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, + "double", strlen("double"))) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, + "%f, ", rand_double()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, + "smallint", strlen("smallint"))) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, + "%d, ", rand_smallint()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, + "tinyint", strlen("tinyint"))) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, + "%d, ", rand_tinyint()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, + "bool", strlen("bool"))) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, + "%d, ", rand_bool()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, + "timestamp", strlen("timestamp"))) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, + "%"PRId64", ", rand_bigint()); } else { printf("No support data type: %s\n", stbInfo->tags[i].dataType); tmfree(dataBuf); return NULL; } } + dataLen -= 2; dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, ")"); return dataBuf; @@ -1871,7 +1936,7 @@ static int calcRowLen(SSuperTable* superTbls) { int lenOfTagOfOneRow = 0; for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) { char* dataType = superTbls->tags[tagIndex].dataType; - + if (strcasecmp(dataType, "BINARY") == 0) { lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; } else if (strcasecmp(dataType, "NCHAR") == 0) { @@ -1902,15 +1967,25 @@ static int calcRowLen(SSuperTable* superTbls) { } -static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName, char* sTblName, char** childTblNameOfSuperTbl, int* childTblCountOfSuperTbl) { +static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, + char* dbName, char* sTblName, char** childTblNameOfSuperTbl, + int* childTblCountOfSuperTbl, int limit, int offset) { + char command[BUFFER_SIZE] = "\0"; + char limitBuf[100] = "\0"; + TAOS_RES * res; TAOS_ROW row = NULL; char* childTblName = *childTblNameOfSuperTbl; - + + if (offset >= 0) { + snprintf(limitBuf, 100, " limit %d offset %d", limit, offset); + } + //get all child table name use cmd: select tbname from superTblName; - snprintf(command, BUFFER_SIZE, "select tbname from %s.%s", dbName, sTblName); + snprintf(command, BUFFER_SIZE, "select tbname from %s.%s %s", dbName, sTblName, limitBuf); + res = taos_query(taos, command); int32_t code = taos_errno(res); if (code != 0) { @@ -1920,7 +1995,7 @@ static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName, char* sTblName exit(-1); } - int childTblCount = 10000; + int childTblCount = (limit < 0)?10000:limit; int count = 0; childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN); char* pTblName = childTblName; @@ -1934,7 +2009,8 @@ static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName, char* sTblName if (tmp != NULL) { childTblName = tmp; childTblCount = (int)(childTblCount*1.5); - memset(childTblName + count*TSDB_TABLE_NAME_LEN, 0, (size_t)((childTblCount-count)*TSDB_TABLE_NAME_LEN)); + memset(childTblName + count*TSDB_TABLE_NAME_LEN, 0, + (size_t)((childTblCount-count)*TSDB_TABLE_NAME_LEN)); } else { // exit, if allocate more memory failed printf("realloc fail for save child table name of %s.%s\n", dbName, sTblName); @@ -1946,7 +2022,7 @@ static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName, char* sTblName } pTblName = childTblName + count * TSDB_TABLE_NAME_LEN; } - + *childTblCountOfSuperTbl = count; *childTblNameOfSuperTbl = childTblName; @@ -1954,19 +2030,29 @@ static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName, char* sTblName return 0; } -static int getSuperTableFromServer(TAOS * taos, char* dbName, SSuperTable* superTbls) { +static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName, + char* sTblName, char** childTblNameOfSuperTbl, + int* childTblCountOfSuperTbl) { + + return getChildNameOfSuperTableWithLimitAndOffset(taos, dbName, sTblName, + childTblNameOfSuperTbl, childTblCountOfSuperTbl, + -1, -1); +} + +static int getSuperTableFromServer(TAOS * taos, char* dbName, + SSuperTable* superTbls) { char command[BUFFER_SIZE] = "\0"; TAOS_RES * res; TAOS_ROW row = NULL; int count = 0; - - //get schema use cmd: describe superTblName; + + //get schema use cmd: describe superTblName; snprintf(command, BUFFER_SIZE, "describe %s.%s", dbName, superTbls->sTblName); - res = taos_query(taos, command); + res = taos_query(taos, command); int32_t code = taos_errno(res); if (code != 0) { printf("failed to run command %s\n", command); - taos_free_result(res); + taos_free_result(res); return -1; } @@ -1977,19 +2063,33 @@ static int getSuperTableFromServer(TAOS * taos, char* dbName, SSuperTable* supe if (0 == count) { count++; continue; - } + } if (strcmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "TAG") == 0) { - tstrncpy(superTbls->tags[tagIndex].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); - tstrncpy(superTbls->tags[tagIndex].dataType, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes); - superTbls->tags[tagIndex].dataLen = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); - tstrncpy(superTbls->tags[tagIndex].note, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); + tstrncpy(superTbls->tags[tagIndex].field, + (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], + fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); + tstrncpy(superTbls->tags[tagIndex].dataType, + (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes); + superTbls->tags[tagIndex].dataLen = + *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); + tstrncpy(superTbls->tags[tagIndex].note, + (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], + fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); tagIndex++; - } else { - tstrncpy(superTbls->columns[columnIndex].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); - tstrncpy(superTbls->columns[columnIndex].dataType, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes); - superTbls->columns[columnIndex].dataLen = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); - tstrncpy(superTbls->columns[columnIndex].note, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); + } else { + tstrncpy(superTbls->columns[columnIndex].field, + (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], + fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); + tstrncpy(superTbls->columns[columnIndex].dataType, + (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes); + superTbls->columns[columnIndex].dataLen = + *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); + tstrncpy(superTbls->columns[columnIndex].note, + (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], + fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); columnIndex++; } count++; @@ -2003,14 +2103,17 @@ static int getSuperTableFromServer(TAOS * taos, char* dbName, SSuperTable* supe if (TBL_ALREADY_EXISTS == superTbls->childTblExists) { //get all child table name use cmd: select tbname from superTblName; - getAllChildNameOfSuperTable(taos, dbName, superTbls->sTblName, &superTbls->childTblName, &superTbls->childTblCount); + getAllChildNameOfSuperTable(taos, dbName, + superTbls->sTblName, + &superTbls->childTblName, + &superTbls->childTblCount); } return 0; } -static int createSuperTable(TAOS * taos, char* dbName, SSuperTable* superTbls, bool use_metric) { +static int createSuperTable(TAOS * taos, char* dbName, SSuperTable* superTbls, bool use_metric) { char command[BUFFER_SIZE] = "\0"; - + char cols[STRING_LEN] = "\0"; int colIndex; int len = 0; @@ -2018,7 +2121,7 @@ static int createSuperTable(TAOS * taos, char* dbName, SSuperTable* superTbls, int lenOfOneRow = 0; for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) { char* dataType = superTbls->columns[colIndex].dataType; - + if (strcasecmp(dataType, "BINARY") == 0) { len += snprintf(cols + len, STRING_LEN - len, ", col%d %s(%d)", colIndex, "BINARY", @@ -2047,10 +2150,10 @@ static int createSuperTable(TAOS * taos, char* dbName, SSuperTable* superTbls, } else if (strcasecmp(dataType, "FLOAT") == 0) { len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "FLOAT"); lenOfOneRow += 22; - } else if (strcasecmp(dataType, "DOUBLE") == 0) { + } else if (strcasecmp(dataType, "DOUBLE") == 0) { len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "DOUBLE"); lenOfOneRow += 42; - } else if (strcasecmp(dataType, "TIMESTAMP") == 0) { + } else if (strcasecmp(dataType, "TIMESTAMP") == 0) { len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TIMESTAMP"); lenOfOneRow += 21; } else { @@ -2082,33 +2185,42 @@ static int createSuperTable(TAOS * taos, char* dbName, SSuperTable* superTbls, len += snprintf(tags + len, STRING_LEN - len, "("); for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) { char* dataType = superTbls->tags[tagIndex].dataType; - + if (strcasecmp(dataType, "BINARY") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex, "BINARY", superTbls->tags[tagIndex].dataLen); + len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex, + "BINARY", superTbls->tags[tagIndex].dataLen); lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; } else if (strcasecmp(dataType, "NCHAR") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex, "NCHAR", superTbls->tags[tagIndex].dataLen); + len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex, + "NCHAR", superTbls->tags[tagIndex].dataLen); lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; } else if (strcasecmp(dataType, "INT") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "INT"); + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, + "INT"); lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11; } else if (strcasecmp(dataType, "BIGINT") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "BIGINT"); + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, + "BIGINT"); lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21; } else if (strcasecmp(dataType, "SMALLINT") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "SMALLINT"); + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, + "SMALLINT"); lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; } else if (strcasecmp(dataType, "TINYINT") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "TINYINT"); + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, + "TINYINT"); lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4; } else if (strcasecmp(dataType, "BOOL") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "BOOL"); + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, + "BOOL"); lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; } else if (strcasecmp(dataType, "FLOAT") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "FLOAT"); + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, + "FLOAT"); lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22; } else if (strcasecmp(dataType, "DOUBLE") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "DOUBLE"); + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, + "DOUBLE"); lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42; } else { taos_close(taos); @@ -2127,7 +2239,8 @@ static int createSuperTable(TAOS * taos, char* dbName, SSuperTable* superTbls, verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, command); if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { - fprintf(stderr, "create supertable %s failed!\n\n", superTbls->sTblName); + fprintf(stderr, "create supertable %s failed!\n\n", + superTbls->sTblName); return -1; } debugPrint("create supertable %s success!\n\n", superTbls->sTblName); @@ -2135,7 +2248,6 @@ static int createSuperTable(TAOS * taos, char* dbName, SSuperTable* superTbls, return 0; } - static int createDatabases() { TAOS * taos = NULL; int ret = 0; @@ -2158,31 +2270,35 @@ static int createDatabases() { int dataLen = 0; dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, "create database if not exists %s ", g_Dbs.db[i].dbName); + BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName); if (g_Dbs.db[i].dbCfg.blocks > 0) { dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, "blocks %d ", g_Dbs.db[i].dbCfg.blocks); + BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks); } if (g_Dbs.db[i].dbCfg.cache > 0) { dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, "cache %d ", g_Dbs.db[i].dbCfg.cache); + BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache); } if (g_Dbs.db[i].dbCfg.days > 0) { dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, "days %d ", g_Dbs.db[i].dbCfg.days); + BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days); } if (g_Dbs.db[i].dbCfg.keep > 0) { dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, "keep %d ", g_Dbs.db[i].dbCfg.keep); + BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep); + } + if (g_Dbs.db[i].dbCfg.quorum > 1) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum); } if (g_Dbs.db[i].dbCfg.replica > 0) { dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, "replica %d ", g_Dbs.db[i].dbCfg.replica); + BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica); } if (g_Dbs.db[i].dbCfg.update > 0) { dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, "update %d ", g_Dbs.db[i].dbCfg.update); + BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update); } //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) { // dataLen += snprintf(command + dataLen, @@ -2190,31 +2306,32 @@ static int createDatabases() { //} if (g_Dbs.db[i].dbCfg.minRows > 0) { dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, "minrows %d ", g_Dbs.db[i].dbCfg.minRows); + BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows); } if (g_Dbs.db[i].dbCfg.maxRows > 0) { dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, "maxrows %d ", g_Dbs.db[i].dbCfg.maxRows); + BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows); } if (g_Dbs.db[i].dbCfg.comp > 0) { dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, "comp %d ", g_Dbs.db[i].dbCfg.comp); + BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp); } if (g_Dbs.db[i].dbCfg.walLevel > 0) { dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, "wal %d ", g_Dbs.db[i].dbCfg.walLevel); + BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel); } if (g_Dbs.db[i].dbCfg.cacheLast > 0) { dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, "cachelast %d ", g_Dbs.db[i].dbCfg.cacheLast); + BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast); } if (g_Dbs.db[i].dbCfg.fsync > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "fsync %d ", g_Dbs.db[i].dbCfg.fsync); + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, + " fsync %d", g_Dbs.db[i].dbCfg.fsync); } - if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) - || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { + if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms"))) + || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", strlen("us")))) { dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, - "precision \'%s\';", g_Dbs.db[i].dbCfg.precision); + " precision \'%s\';", g_Dbs.db[i].dbCfg.precision); } debugPrint("%s() %d command: %s\n", __func__, __LINE__, command); @@ -2344,7 +2461,7 @@ static void* createTable(void *sarg) } int startMultiThreadCreateChildTable( - char* cols, int threads, int ntables, + char* cols, int threads, int startFrom, int ntables, char* db_name, SSuperTable* superTblInfo) { pthread_t *pids = malloc(threads * sizeof(pthread_t)); threadInfo *infos = malloc(threads * sizeof(threadInfo)); @@ -2367,7 +2484,7 @@ int startMultiThreadCreateChildTable( int b = 0; b = ntables % threads; - int last = 0; + int last = startFrom; for (int i = 0; i < threads; i++) { threadInfo *t_info = infos + i; t_info->threadID = i; @@ -2415,7 +2532,7 @@ static void createChildTables() { char tblColsBuf[MAX_SQL_SIZE]; int len; - for (int i = 0; i < g_Dbs.dbCount; i++) { + for (int i = 0; i < g_Dbs.dbCount; i++) { if (g_Dbs.db[i].superTblCount > 0) { // with super table for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { @@ -2426,12 +2543,17 @@ static void createChildTables() { verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, g_Dbs.db[i].superTbls[j].colsOfCreateChildTable); + int startFrom = 0; + g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount; + + verbosePrint("%s() LN%d: create %d child tables from %d\n", __func__, __LINE__, + g_totalChildTables, startFrom); startMultiThreadCreateChildTable( g_Dbs.db[i].superTbls[j].colsOfCreateChildTable, g_Dbs.threadCountByCreateTbl, - g_Dbs.db[i].superTbls[j].childTblCount, + startFrom, + g_totalChildTables, g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j])); - g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount; } } else { // normal table @@ -2439,10 +2561,13 @@ static void createChildTables() { int j = 0; while (g_args.datatype[j]) { if ((strncasecmp(g_args.datatype[j], "BINARY", strlen("BINARY")) == 0) - || (strncasecmp(g_args.datatype[j], "NCHAR", strlen("NCHAR")) == 0)) { - len = snprintf(tblColsBuf + len, MAX_SQL_SIZE, ", COL%d %s(60)", j, g_args.datatype[j]); + || (strncasecmp(g_args.datatype[j], + "NCHAR", strlen("NCHAR")) == 0)) { + len = snprintf(tblColsBuf + len, MAX_SQL_SIZE, + ", COL%d %s(60)", j, g_args.datatype[j]); } else { - len = snprintf(tblColsBuf + len, MAX_SQL_SIZE, ", COL%d %s", j, g_args.datatype[j]); + len = snprintf(tblColsBuf + len, MAX_SQL_SIZE, + ", COL%d %s", j, g_args.datatype[j]); } len = strlen(tblColsBuf); j++; @@ -2450,11 +2575,13 @@ static void createChildTables() { len = snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")"); - verbosePrint("%s() LN%d: dbName: %s num of tb: %d schema: %s\n", __func__, __LINE__, + verbosePrint("%s() LN%d: dbName: %s num of tb: %d schema: %s\n", + __func__, __LINE__, g_Dbs.db[i].dbName, g_args.num_of_tables, tblColsBuf); startMultiThreadCreateChildTable( tblColsBuf, g_Dbs.threadCountByCreateTbl, + 0, g_args.num_of_tables, g_Dbs.db[i].dbName, NULL); @@ -2462,30 +2589,6 @@ static void createChildTables() { } } -/* -static int taosGetLineNum(const char *fileName) -{ - int lineNum = 0; - char cmd[1024] = { 0 }; - char buf[1024] = { 0 }; - sprintf(cmd, "wc -l %s", fileName); - - FILE *fp = popen(cmd, "r"); - if (fp == NULL) { - fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno)); - return lineNum; - } - - if (fgets(buf, sizeof(buf), fp)) { - int index = strchr((const char*)buf, ' ') - buf; - buf[index] = '\0'; - lineNum = atoi(buf); - } - pclose(fp); - return lineNum; -} -*/ - /* Read 10000 lines at most. If more than 10000 lines, continue to read after using */ @@ -2563,19 +2666,29 @@ int readSampleFromJsonFileToMem(SSuperTable * superTblInfo) { /* Read 10000 lines at most. If more than 10000 lines, continue to read after using */ -int readSampleFromCsvFileToMem(FILE *fp, SSuperTable* superTblInfo, char* sampleBuf) { +static int readSampleFromCsvFileToMem( + SSuperTable* superTblInfo) { size_t n = 0; ssize_t readLen = 0; char * line = NULL; int getRows = 0; + + FILE* fp = fopen(superTblInfo->sampleFile, "r"); + if (fp == NULL) { + fprintf(stderr, "Failed to open sample file: %s, reason:%s\n", + superTblInfo->sampleFile, strerror(errno)); + return -1; + } - memset(sampleBuf, 0, MAX_SAMPLES_ONCE_FROM_FILE* superTblInfo->lenOfOneRow); + memset(superTblInfo->sampleDataBuf, 0, + MAX_SAMPLES_ONCE_FROM_FILE * superTblInfo->lenOfOneRow); while (1) { readLen = tgetline(&line, &n, fp); if (-1 == readLen) { if(0 != fseek(fp, 0, SEEK_SET)) { - printf("Failed to fseek file: %s, reason:%s\n", + fprintf(stderr, "Failed to fseek file: %s, reason:%s\n", superTblInfo->sampleFile, strerror(errno)); + fclose(fp); return -1; } continue; @@ -2595,7 +2708,8 @@ int readSampleFromCsvFileToMem(FILE *fp, SSuperTable* superTblInfo, char* sample continue; } - memcpy(sampleBuf + getRows * superTblInfo->lenOfOneRow, line, readLen); + memcpy(superTblInfo->sampleDataBuf + getRows * superTblInfo->lenOfOneRow, + line, readLen); getRows++; if (getRows == MAX_SAMPLES_ONCE_FROM_FILE) { @@ -2603,6 +2717,7 @@ int readSampleFromCsvFileToMem(FILE *fp, SSuperTable* superTblInfo, char* sample } } + fclose(fp); tmfree(line); return 0; } @@ -2627,7 +2742,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile(cJSON* stbInfo, SSuperTable* s // columns cJSON *columns = cJSON_GetObjectItem(stbInfo, "columns"); if (columns && columns->type != cJSON_Array) { - printf("failed to read json, columns not found\n"); + printf("ERROR: failed to read json, columns not found\n"); goto PARSE_OVER; } else if (NULL == columns) { superTbls->columnCount = 0; @@ -2637,7 +2752,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile(cJSON* stbInfo, SSuperTable* s int columnSize = cJSON_GetArraySize(columns); if (columnSize > MAX_COLUMN_COUNT) { - printf("failed to read json, column size overflow, max column size is %d\n", + printf("ERROR: failed to read json, column size overflow, max column size is %d\n", MAX_COLUMN_COUNT); goto PARSE_OVER; } @@ -2656,7 +2771,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile(cJSON* stbInfo, SSuperTable* s if (countObj && countObj->type == cJSON_Number) { count = countObj->valueint; } else if (countObj && countObj->type != cJSON_Number) { - printf("failed to read json, column count not found"); + printf("ERROR: failed to read json, column count not found\n"); goto PARSE_OVER; } else { count = 1; @@ -2666,7 +2781,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile(cJSON* stbInfo, SSuperTable* s memset(&columnCase, 0, sizeof(StrColumn)); cJSON *dataType = cJSON_GetObjectItem(column, "type"); if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) { - printf("failed to read json, column type not found"); + printf("ERROR: failed to read json, column type not found\n"); goto PARSE_OVER; } //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE); @@ -2676,7 +2791,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile(cJSON* stbInfo, SSuperTable* s if (dataLen && dataLen->type == cJSON_Number) { columnCase.dataLen = dataLen->valueint; } else if (dataLen && dataLen->type != cJSON_Number) { - printf("failed to read json, column len not found"); + printf("ERROR: failed to read json, column len not found\n"); goto PARSE_OVER; } else { columnCase.dataLen = 8; @@ -2695,13 +2810,13 @@ static bool getColumnAndTagTypeFromInsertJsonFile(cJSON* stbInfo, SSuperTable* s // tags cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags"); if (!tags || tags->type != cJSON_Array) { - printf("failed to read json, tags not found"); + printf("ERROR: failed to read json, tags not found\n"); goto PARSE_OVER; } int tagSize = cJSON_GetArraySize(tags); if (tagSize > MAX_TAG_COUNT) { - printf("failed to read json, tags size overflow, max tag size is %d\n", MAX_TAG_COUNT); + printf("ERROR: failed to read json, tags size overflow, max tag size is %d\n", MAX_TAG_COUNT); goto PARSE_OVER; } @@ -2715,7 +2830,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile(cJSON* stbInfo, SSuperTable* s if (countObj && countObj->type == cJSON_Number) { count = countObj->valueint; } else if (countObj && countObj->type != cJSON_Number) { - printf("failed to read json, column count not found"); + printf("ERROR: failed to read json, column count not found\n"); goto PARSE_OVER; } else { count = 1; @@ -2725,7 +2840,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile(cJSON* stbInfo, SSuperTable* s memset(&columnCase, 0, sizeof(StrColumn)); cJSON *dataType = cJSON_GetObjectItem(tag, "type"); if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) { - printf("failed to read json, tag type not found"); + printf("ERROR: failed to read json, tag type not found\n"); goto PARSE_OVER; } tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE); @@ -2734,7 +2849,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile(cJSON* stbInfo, SSuperTable* s if (dataLen && dataLen->type == cJSON_Number) { columnCase.dataLen = dataLen->valueint; } else if (dataLen && dataLen->type != cJSON_Number) { - printf("failed to read json, column len not found"); + printf("ERROR: failed to read json, column len not found\n"); goto PARSE_OVER; } else { columnCase.dataLen = 0; @@ -2771,7 +2886,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!host) { tstrncpy(g_Dbs.host, "127.0.0.1", MAX_DB_NAME_SIZE); } else { - printf("failed to read json, host not found\n"); + printf("ERROR: failed to read json, host not found\n"); goto PARSE_OVER; } @@ -2809,37 +2924,48 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!threads) { g_Dbs.threadCount = 1; } else { - printf("failed to read json, threads not found"); + printf("ERROR: failed to read json, threads not found\n"); goto PARSE_OVER; } - + cJSON* threads2 = cJSON_GetObjectItem(root, "thread_count_create_tbl"); if (threads2 && threads2->type == cJSON_Number) { g_Dbs.threadCountByCreateTbl = threads2->valueint; } else if (!threads2) { g_Dbs.threadCountByCreateTbl = 1; } else { - printf("failed to read json, threads2 not found"); + printf("ERROR: failed to read json, threads2 not found\n"); goto PARSE_OVER; } - cJSON* insertInterval = cJSON_GetObjectItem(root, "insert_interval"); - if (insertInterval && insertInterval->type == cJSON_Number) { - g_args.insert_interval = insertInterval->valueint; - } else if (!insertInterval) { + cJSON* gInsertInterval = cJSON_GetObjectItem(root, "insert_interval"); + if (gInsertInterval && gInsertInterval->type == cJSON_Number) { + g_args.insert_interval = gInsertInterval->valueint; + } else if (!gInsertInterval) { g_args.insert_interval = 0; } else { - printf("failed to read json, insert_interval not found"); + fprintf(stderr, "ERROR: failed to read json, insert_interval input mistake\n"); goto PARSE_OVER; } + cJSON* maxSqlLen = cJSON_GetObjectItem(root, "max_sql_len"); + if (maxSqlLen && maxSqlLen->type == cJSON_Number) { + g_args.max_sql_len = maxSqlLen->valueint; + } else if (!maxSqlLen) { + g_args.max_sql_len = TSDB_PAYLOAD_SIZE; + } else { + fprintf(stderr, "ERROR: failed to read json, max_sql_len input mistake\n"); + goto PARSE_OVER; + } + + cJSON* numRecPerReq = cJSON_GetObjectItem(root, "num_of_records_per_req"); if (numRecPerReq && numRecPerReq->type == cJSON_Number) { g_args.num_of_RPR = numRecPerReq->valueint; } else if (!numRecPerReq) { g_args.num_of_RPR = 100; } else { - printf("failed to read json, num_of_records_per_req not found"); + printf("ERROR: failed to read json, num_of_records_per_req not found\n"); goto PARSE_OVER; } @@ -2857,19 +2983,19 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!answerPrompt) { g_args.answer_yes = false; } else { - printf("failed to read json, confirm_parameter_prompt not found"); + printf("ERROR: failed to read json, confirm_parameter_prompt not found\n"); goto PARSE_OVER; } cJSON* dbs = cJSON_GetObjectItem(root, "databases"); if (!dbs || dbs->type != cJSON_Array) { - printf("failed to read json, databases not found\n"); + printf("ERROR: failed to read json, databases not found\n"); goto PARSE_OVER; } int dbSize = cJSON_GetArraySize(dbs); if (dbSize > MAX_DB_COUNT) { - printf("failed to read json, databases size overflow, max database is %d\n", MAX_DB_COUNT); + printf("ERROR: failed to read json, databases size overflow, max database is %d\n", MAX_DB_COUNT); goto PARSE_OVER; } @@ -2881,13 +3007,13 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { // dbinfo cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo"); if (!dbinfo || dbinfo->type != cJSON_Object) { - printf("failed to read json, dbinfo not found"); + printf("ERROR: failed to read json, dbinfo not found\n"); goto PARSE_OVER; } cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name"); if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) { - printf("failed to read json, db name not found"); + printf("ERROR: failed to read json, db name not found\n"); goto PARSE_OVER; } tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, MAX_DB_NAME_SIZE); @@ -2902,7 +3028,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!drop) { g_Dbs.db[i].drop = 0; } else { - printf("failed to read json, drop not found"); + printf("ERROR: failed to read json, drop not found\n"); goto PARSE_OVER; } @@ -2913,7 +3039,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { //tstrncpy(g_Dbs.db[i].dbCfg.precision, "ms", MAX_DB_NAME_SIZE); memset(g_Dbs.db[i].dbCfg.precision, 0, MAX_DB_NAME_SIZE); } else { - printf("failed to read json, precision not found"); + printf("ERROR: failed to read json, precision not found\n"); goto PARSE_OVER; } @@ -2923,7 +3049,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!update) { g_Dbs.db[i].dbCfg.update = -1; } else { - printf("failed to read json, update not found"); + printf("ERROR: failed to read json, update not found\n"); goto PARSE_OVER; } @@ -2933,7 +3059,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!replica) { g_Dbs.db[i].dbCfg.replica = -1; } else { - printf("failed to read json, replica not found"); + printf("ERROR: failed to read json, replica not found\n"); goto PARSE_OVER; } @@ -2943,7 +3069,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!keep) { g_Dbs.db[i].dbCfg.keep = -1; } else { - printf("failed to read json, keep not found"); + printf("ERROR: failed to read json, keep not found\n"); goto PARSE_OVER; } @@ -2953,7 +3079,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!days) { g_Dbs.db[i].dbCfg.days = -1; } else { - printf("failed to read json, days not found"); + printf("ERROR: failed to read json, days not found\n"); goto PARSE_OVER; } @@ -2963,7 +3089,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!cache) { g_Dbs.db[i].dbCfg.cache = -1; } else { - printf("failed to read json, cache not found"); + printf("ERROR: failed to read json, cache not found\n"); goto PARSE_OVER; } @@ -2973,7 +3099,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!blocks) { g_Dbs.db[i].dbCfg.blocks = -1; } else { - printf("failed to read json, block not found"); + printf("ERROR: failed to read json, block not found\n"); goto PARSE_OVER; } @@ -2993,7 +3119,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!minRows) { g_Dbs.db[i].dbCfg.minRows = -1; } else { - printf("failed to read json, minRows not found"); + printf("ERROR: failed to read json, minRows not found\n"); goto PARSE_OVER; } @@ -3003,7 +3129,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!maxRows) { g_Dbs.db[i].dbCfg.maxRows = -1; } else { - printf("failed to read json, maxRows not found"); + printf("ERROR: failed to read json, maxRows not found\n"); goto PARSE_OVER; } @@ -3013,7 +3139,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!comp) { g_Dbs.db[i].dbCfg.comp = -1; } else { - printf("failed to read json, comp not found"); + printf("ERROR: failed to read json, comp not found\n"); goto PARSE_OVER; } @@ -3023,7 +3149,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!walLevel) { g_Dbs.db[i].dbCfg.walLevel = -1; } else { - printf("failed to read json, walLevel not found"); + printf("ERROR: failed to read json, walLevel not found\n"); goto PARSE_OVER; } @@ -3033,7 +3159,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!cacheLast) { g_Dbs.db[i].dbCfg.cacheLast = -1; } else { - printf("failed to read json, cacheLast not found"); + printf("ERROR: failed to read json, cacheLast not found\n"); goto PARSE_OVER; } @@ -3041,9 +3167,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (quorum && quorum->type == cJSON_Number) { g_Dbs.db[i].dbCfg.quorum = quorum->valueint; } else if (!quorum) { - g_Dbs.db[i].dbCfg.quorum = -1; + g_Dbs.db[i].dbCfg.quorum = 1; } else { - printf("failed to read json, walLevel not found"); + printf("failed to read json, quorum input mistake"); goto PARSE_OVER; } @@ -3053,20 +3179,20 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!fsync) { g_Dbs.db[i].dbCfg.fsync = -1; } else { - printf("failed to read json, fsync not found"); + printf("ERROR: failed to read json, fsync not found\n"); goto PARSE_OVER; } // super_talbes cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables"); if (!stables || stables->type != cJSON_Array) { - printf("failed to read json, super_tables not found"); + printf("ERROR: failed to read json, super_tables not found\n"); goto PARSE_OVER; } int stbSize = cJSON_GetArraySize(stables); if (stbSize > MAX_SUPER_TABLE_COUNT) { - printf("failed to read json, databases size overflow, max database is %d\n", MAX_SUPER_TABLE_COUNT); + printf("ERROR: failed to read json, databases size overflow, max database is %d\n", MAX_SUPER_TABLE_COUNT); goto PARSE_OVER; } @@ -3078,14 +3204,14 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { // dbinfo cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name"); if (!stbName || stbName->type != cJSON_String || stbName->valuestring == NULL) { - printf("failed to read json, stb name not found"); + printf("ERROR: failed to read json, stb name not found\n"); goto PARSE_OVER; } tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, MAX_TB_NAME_SIZE); cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix"); if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) { - printf("failed to read json, childtable_prefix not found"); + printf("ERROR: failed to read json, childtable_prefix not found\n"); goto PARSE_OVER; } tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, MAX_DB_NAME_SIZE); @@ -3104,7 +3230,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!autoCreateTbl) { g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; } else { - printf("failed to read json, auto_create_table not found"); + printf("ERROR: failed to read json, auto_create_table not found\n"); goto PARSE_OVER; } @@ -3114,7 +3240,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!batchCreateTbl) { g_Dbs.db[i].superTbls[j].batchCreateTableNum = 1000; } else { - printf("failed to read json, batch_create_tbl_num not found"); + printf("ERROR: failed to read json, batch_create_tbl_num not found\n"); goto PARSE_OVER; } @@ -3132,13 +3258,13 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!childTblExists) { g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; } else { - printf("failed to read json, child_table_exists not found"); + printf("ERROR: failed to read json, child_table_exists not found\n"); goto PARSE_OVER; } cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count"); if (!count || count->type != cJSON_Number || 0 >= count->valueint) { - printf("failed to read json, childtable_count not found"); + printf("ERROR: failed to read json, childtable_count not found\n"); goto PARSE_OVER; } g_Dbs.db[i].superTbls[j].childTblCount = count->valueint; @@ -3151,7 +3277,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!dataSource) { tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", MAX_DB_NAME_SIZE); } else { - printf("failed to read json, data_source not found"); + printf("ERROR: failed to read json, data_source not found\n"); goto PARSE_OVER; } @@ -3163,27 +3289,49 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!insertMode) { tstrncpy(g_Dbs.db[i].superTbls[j].insertMode, "taosc", MAX_DB_NAME_SIZE); } else { - printf("failed to read json, insert_mode not found"); + printf("ERROR: failed to read json, insert_mode not found\n"); goto PARSE_OVER; } + cJSON* childTbl_limit = cJSON_GetObjectItem(stbInfo, "childtable_limit"); + if (childTbl_limit) { + if (childTbl_limit->type != cJSON_Number) { + printf("ERROR: failed to read json, childtable_limit\n"); + goto PARSE_OVER; + } + g_Dbs.db[i].superTbls[j].childTblLimit = childTbl_limit->valueint; + } else { + g_Dbs.db[i].superTbls[j].childTblLimit = -1; // select ... limit -1 means all query result + } + + cJSON* childTbl_offset = cJSON_GetObjectItem(stbInfo, "childtable_offset"); + if (childTbl_offset) { + if (childTbl_offset->type != cJSON_Number || 0 > childTbl_offset->valueint) { + printf("ERROR: failed to read json, childtable_offset\n"); + goto PARSE_OVER; + } + g_Dbs.db[i].superTbls[j].childTblOffset = childTbl_offset->valueint; + } else { + g_Dbs.db[i].superTbls[j].childTblOffset = 0; + } + cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp"); if (ts && ts->type == cJSON_String && ts->valuestring != NULL) { tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp, ts->valuestring, MAX_DB_NAME_SIZE); } else if (!ts) { tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp, "now", MAX_DB_NAME_SIZE); } else { - printf("failed to read json, start_timestamp not found"); + printf("ERROR: failed to read json, start_timestamp not found\n"); goto PARSE_OVER; } - + cJSON* timestampStep = cJSON_GetObjectItem(stbInfo, "timestamp_step"); if (timestampStep && timestampStep->type == cJSON_Number) { g_Dbs.db[i].superTbls[j].timeStampStep = timestampStep->valueint; } else if (!timestampStep) { - g_Dbs.db[i].superTbls[j].timeStampStep = 1000; + g_Dbs.db[i].superTbls[j].timeStampStep = DEFAULT_TIMESTAMP_STEP; } else { - printf("failed to read json, timestamp_step not found"); + printf("ERROR: failed to read json, timestamp_step not found\n"); goto PARSE_OVER; } @@ -3196,10 +3344,10 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!sampleDataBufSize) { g_Dbs.db[i].superTbls[j].sampleDataBufSize = 1024*1024 + 1024; } else { - printf("failed to read json, sample_buf_size not found"); + printf("ERROR: failed to read json, sample_buf_size not found\n"); goto PARSE_OVER; } - + cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format"); if (sampleFormat && sampleFormat->type == cJSON_String && sampleFormat->valuestring != NULL) { tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, @@ -3207,10 +3355,10 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!sampleFormat) { tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", MAX_DB_NAME_SIZE); } else { - printf("failed to read json, sample_format not found"); + printf("ERROR: failed to read json, sample_format not found\n"); goto PARSE_OVER; } - + cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file"); if (sampleFile && sampleFile->type == cJSON_String && sampleFile->valuestring != NULL) { tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile, @@ -3218,10 +3366,10 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!sampleFile) { memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN); } else { - printf("failed to read json, sample_file not found"); + printf("ERROR: failed to read json, sample_file not found\n"); goto PARSE_OVER; } - + cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file"); if (tagsFile && tagsFile->type == cJSON_String && tagsFile->valuestring != NULL) { tstrncpy(g_Dbs.db[i].superTbls[j].tagsFile, @@ -3235,10 +3383,10 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { memset(g_Dbs.db[i].superTbls[j].tagsFile, 0, MAX_FILE_NAME_LEN); g_Dbs.db[i].superTbls[j].tagSource = 0; } else { - printf("failed to read json, tags_file not found"); + printf("ERROR: failed to read json, tags_file not found\n"); goto PARSE_OVER; } - + cJSON* maxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len"); if (maxSqlLen && maxSqlLen->type == cJSON_Number) { int32_t len = maxSqlLen->valueint; @@ -3251,7 +3399,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!maxSqlLen) { g_Dbs.db[i].superTbls[j].maxSqlLen = TSDB_MAX_SQL_LEN; } else { - printf("failed to read json, maxSqlLen not found"); + printf("ERROR: failed to read json, maxSqlLen not found\n"); goto PARSE_OVER; } @@ -3268,7 +3416,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!multiThreadWriteOneTbl) { g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0; } else { - printf("failed to read json, multiThreadWriteOneTbl not found"); + printf("ERROR: failed to read json, multiThreadWriteOneTbl not found\n"); goto PARSE_OVER; } @@ -3278,7 +3426,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!numberOfTblInOneSql) { g_Dbs.db[i].superTbls[j].numberOfTblInOneSql = 0; } else { - printf("failed to read json, numberOfTblInOneSql not found"); + printf("ERROR: failed to read json, numberOfTblInOneSql not found\n"); goto PARSE_OVER; } @@ -3288,7 +3436,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!rowsPerTbl) { g_Dbs.db[i].superTbls[j].rowsPerTbl = 1; } else { - printf("failed to read json, rowsPerTbl not found"); + printf("ERROR: failed to read json, rowsPerTbl not found\n"); goto PARSE_OVER; } @@ -3298,7 +3446,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!disorderRatio) { g_Dbs.db[i].superTbls[j].disorderRatio = 0; } else { - printf("failed to read json, disorderRatio not found"); + printf("ERROR: failed to read json, disorderRatio not found\n"); goto PARSE_OVER; } @@ -3308,20 +3456,29 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!disorderRange) { g_Dbs.db[i].superTbls[j].disorderRange = 1000; } else { - printf("failed to read json, disorderRange not found"); + printf("ERROR: failed to read json, disorderRange not found\n"); goto PARSE_OVER; } cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows"); if (insertRows && insertRows->type == cJSON_Number) { g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint; - //if (0 == g_Dbs.db[i].superTbls[j].insertRows) { - // g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF; - //} } else if (!insertRows) { g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF; } else { - printf("failed to read json, insert_rows not found"); + fprintf(stderr, "failed to read json, insert_rows input mistake"); + goto PARSE_OVER; + } + + cJSON* insertInterval = cJSON_GetObjectItem(stbInfo, "insert_interval"); + if (insertInterval && insertInterval->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint; + } else if (!insertInterval) { + debugPrint("%s() LN%d: stable insert interval be overrided by global %d.\n", + __func__, __LINE__, g_args.insert_interval); + g_Dbs.db[i].superTbls[j].insertInterval = g_args.insert_interval; + } else { + fprintf(stderr, "failed to read json, insert_interval input mistake"); goto PARSE_OVER; } @@ -3360,7 +3517,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else if (!host) { tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_DB_NAME_SIZE); } else { - printf("failed to read json, host not found\n"); + printf("ERROR: failed to read json, host not found\n"); goto PARSE_OVER; } @@ -3398,7 +3555,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else if (!answerPrompt) { g_args.answer_yes = false; } else { - printf("failed to read json, confirm_parameter_prompt not found"); + printf("ERROR: failed to read json, confirm_parameter_prompt not found\n"); goto PARSE_OVER; } @@ -3406,7 +3563,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) { tstrncpy(g_queryInfo.dbName, dbs->valuestring, MAX_DB_NAME_SIZE); } else if (!dbs) { - printf("failed to read json, databases not found\n"); + printf("ERROR: failed to read json, databases not found\n"); goto PARSE_OVER; } @@ -3416,7 +3573,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else if (!queryMode) { tstrncpy(g_queryInfo.queryMode, "taosc", MAX_TB_NAME_SIZE); } else { - printf("failed to read json, query_mode not found\n"); + printf("ERROR: failed to read json, query_mode not found\n"); goto PARSE_OVER; } @@ -3426,7 +3583,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { g_queryInfo.superQueryInfo.concurrent = 0; g_queryInfo.superQueryInfo.sqlCount = 0; } else if (superQuery->type != cJSON_Object) { - printf("failed to read json, super_table_query not found"); + printf("ERROR: failed to read json, super_table_query not found\n"); goto PARSE_OVER; } else { cJSON* rate = cJSON_GetObjectItem(superQuery, "query_interval"); @@ -3450,7 +3607,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else if (0 == strcmp("async", mode->valuestring)) { g_queryInfo.superQueryInfo.subscribeMode = 1; } else { - printf("failed to read json, subscribe mod error\n"); + printf("ERROR: failed to read json, subscribe mod error\n"); goto PARSE_OVER; } } else { @@ -3473,7 +3630,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else if (0 == strcmp("no", restart->valuestring)) { g_queryInfo.superQueryInfo.subscribeRestart = 0; } else { - printf("failed to read json, subscribe restart error\n"); + printf("ERROR: failed to read json, subscribe restart error\n"); goto PARSE_OVER; } } else { @@ -3489,7 +3646,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else if (0 == strcmp("no", keepProgress->valuestring)) { g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; } else { - printf("failed to read json, subscribe keepProgress error\n"); + printf("ERROR: failed to read json, subscribe keepProgress error\n"); goto PARSE_OVER; } } else { @@ -3501,15 +3658,15 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { if (!superSqls) { g_queryInfo.superQueryInfo.sqlCount = 0; } else if (superSqls->type != cJSON_Array) { - printf("failed to read json, super sqls not found\n"); + printf("ERROR: failed to read json, super sqls not found\n"); goto PARSE_OVER; } else { int superSqlSize = cJSON_GetArraySize(superSqls); if (superSqlSize > MAX_QUERY_SQL_COUNT) { - printf("failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT); + printf("ERROR: failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT); goto PARSE_OVER; } - + g_queryInfo.superQueryInfo.sqlCount = superSqlSize; for (int j = 0; j < superSqlSize; ++j) { cJSON* sql = cJSON_GetArrayItem(superSqls, j); @@ -3517,7 +3674,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) { - printf("failed to read json, sql not found\n"); + printf("ERROR: failed to read json, sql not found\n"); goto PARSE_OVER; } tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); @@ -3528,20 +3685,20 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else if (NULL == result) { memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN); } else { - printf("failed to read json, super query result file not found\n"); + printf("ERROR: failed to read json, super query result file not found\n"); goto PARSE_OVER; } } } } - + // sub_table_query cJSON *subQuery = cJSON_GetObjectItem(root, "super_table_query"); if (!subQuery) { g_queryInfo.subQueryInfo.threadCnt = 0; g_queryInfo.subQueryInfo.sqlCount = 0; } else if (subQuery->type != cJSON_Object) { - printf("failed to read json, sub_table_query not found"); + printf("ERROR: failed to read json, sub_table_query not found\n"); ret = true; goto PARSE_OVER; } else { @@ -3551,29 +3708,29 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else if (!subrate) { g_queryInfo.subQueryInfo.rate = 0; } - + cJSON* threads = cJSON_GetObjectItem(subQuery, "threads"); if (threads && threads->type == cJSON_Number) { g_queryInfo.subQueryInfo.threadCnt = threads->valueint; } else if (!threads) { g_queryInfo.subQueryInfo.threadCnt = 1; } - + //cJSON* subTblCnt = cJSON_GetObjectItem(subQuery, "childtable_count"); //if (subTblCnt && subTblCnt->type == cJSON_Number) { // g_queryInfo.subQueryInfo.childTblCount = subTblCnt->valueint; //} else if (!subTblCnt) { // g_queryInfo.subQueryInfo.childTblCount = 0; //} - + cJSON* stblname = cJSON_GetObjectItem(subQuery, "stblname"); if (stblname && stblname->type == cJSON_String && stblname->valuestring != NULL) { tstrncpy(g_queryInfo.subQueryInfo.sTblName, stblname->valuestring, MAX_TB_NAME_SIZE); } else { - printf("failed to read json, super table name not found\n"); + printf("ERROR: failed to read json, super table name not found\n"); goto PARSE_OVER; } - + cJSON* submode = cJSON_GetObjectItem(subQuery, "mode"); if (submode && submode->type == cJSON_String && submode->valuestring != NULL) { if (0 == strcmp("sync", submode->valuestring)) { @@ -3581,13 +3738,13 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else if (0 == strcmp("async", submode->valuestring)) { g_queryInfo.subQueryInfo.subscribeMode = 1; } else { - printf("failed to read json, subscribe mod error\n"); + printf("ERROR: failed to read json, subscribe mod error\n"); goto PARSE_OVER; } } else { g_queryInfo.subQueryInfo.subscribeMode = 0; } - + cJSON* subinterval = cJSON_GetObjectItem(subQuery, "interval"); if (subinterval && subinterval->type == cJSON_Number) { g_queryInfo.subQueryInfo.subscribeInterval = subinterval->valueint; @@ -3604,7 +3761,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else if (0 == strcmp("no", subrestart->valuestring)) { g_queryInfo.subQueryInfo.subscribeRestart = 0; } else { - printf("failed to read json, subscribe restart error\n"); + printf("ERROR: failed to read json, subscribe restart error\n"); goto PARSE_OVER; } } else { @@ -3618,7 +3775,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else if (0 == strcmp("no", subkeepProgress->valuestring)) { g_queryInfo.subQueryInfo.subscribeKeepProgress = 0; } else { - printf("failed to read json, subscribe keepProgress error\n"); + printf("ERROR: failed to read json, subscribe keepProgress error\n"); goto PARSE_OVER; } } else { @@ -3630,12 +3787,12 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { if (!subsqls) { g_queryInfo.subQueryInfo.sqlCount = 0; } else if (subsqls->type != cJSON_Array) { - printf("failed to read json, super sqls not found\n"); + printf("ERROR: failed to read json, super sqls not found\n"); goto PARSE_OVER; } else { int superSqlSize = cJSON_GetArraySize(subsqls); if (superSqlSize > MAX_QUERY_SQL_COUNT) { - printf("failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT); + printf("ERROR: failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT); goto PARSE_OVER; } @@ -3646,7 +3803,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) { - printf("failed to read json, sql not found\n"); + printf("ERROR: failed to read json, sql not found\n"); goto PARSE_OVER; } tstrncpy(g_queryInfo.subQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); @@ -3657,7 +3814,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else if (NULL == result) { memset(g_queryInfo.subQueryInfo.result[j], 0, MAX_FILE_NAME_LEN); } else { - printf("failed to read json, sub query result file not found\n"); + printf("ERROR: failed to read json, sub query result file not found\n"); goto PARSE_OVER; } } @@ -3696,7 +3853,7 @@ static bool getInfoFromJsonFile(char* file) { content[len] = 0; cJSON* root = cJSON_Parse(content); if (root == NULL) { - printf("failed to cjson parse %s, invalid json format", file); + printf("ERROR: failed to cjson parse %s, invalid json format\n", file); goto PARSE_OVER; } @@ -3709,13 +3866,13 @@ static bool getInfoFromJsonFile(char* file) { } else if (0 == strcasecmp("subscribe", filetype->valuestring)) { g_args.test_mode = SUBSCRIBE_MODE; } else { - printf("failed to read json, filetype not support\n"); + printf("ERROR: failed to read json, filetype not support\n"); goto PARSE_OVER; } } else if (!filetype) { g_args.test_mode = INSERT_MODE; } else { - printf("failed to read json, filetype not found\n"); + printf("ERROR: failed to read json, filetype not found\n"); goto PARSE_OVER; } @@ -3726,7 +3883,7 @@ static bool getInfoFromJsonFile(char* file) { } else if (SUBSCRIBE_MODE == g_args.test_mode) { ret = getMetaFromQueryJsonFile(root); } else { - printf("input json file type error! please input correct file type: insert or query or subscribe\n"); + printf("ERROR: input json file type error! please input correct file type: insert or query or subscribe\n"); goto PARSE_OVER; } @@ -3737,7 +3894,7 @@ PARSE_OVER: return ret; } -void prePareSampleData() { +void prepareSampleData() { for (int i = 0; i < g_Dbs.dbCount; i++) { for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { //if (0 == strncasecmp(g_Dbs.db[i].superTbls[j].dataSource, "sample", 6)) { @@ -3775,22 +3932,26 @@ void postFreeResource() { } } -int getRowDataFromSample(char* dataBuf, int maxLen, int64_t timestamp, SSuperTable* superTblInfo, int* sampleUsePos, FILE *fp, char* sampleBuf) { +static int getRowDataFromSample(char* dataBuf, int maxLen, int64_t timestamp, + SSuperTable* superTblInfo, int* sampleUsePos) { if ((*sampleUsePos) == MAX_SAMPLES_ONCE_FROM_FILE) { - int ret = readSampleFromCsvFileToMem(fp, superTblInfo, sampleBuf); + int ret = readSampleFromCsvFileToMem(superTblInfo); if (0 != ret) { + tmfree(superTblInfo->sampleDataBuf); return -1; } *sampleUsePos = 0; } int dataLen = 0; - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "(%" PRId64 ", ", timestamp); - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%s", sampleBuf + superTblInfo->lenOfOneRow * (*sampleUsePos)); + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, + "(%" PRId64 ", ", timestamp); + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, + "%s", superTblInfo->sampleDataBuf + superTblInfo->lenOfOneRow * (*sampleUsePos)); dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")"); (*sampleUsePos)++; - + return dataLen; } @@ -3841,14 +4002,12 @@ int generateRowData(char* dataBuf, int maxLen, int64_t timestamp, SSuperTable* } static void syncWriteForNumberOfTblInOneSql( - threadInfo *winfo, FILE *fp, char* sampleDataBuf) { + threadInfo *winfo, char* sampleDataBuf) { SSuperTable* superTblInfo = winfo->superTblInfo; int samplePos = 0; //printf("========threadID[%d], table rang: %d - %d \n", winfo->threadID, winfo->start_table_id, winfo->end_table_id); - int64_t totalRowsInserted = 0; - int64_t totalAffectedRows = 0; int64_t lastPrintTime = taosGetTimestampMs(); char* buffer = calloc(superTblInfo->maxSqlLen+1, 1); @@ -3866,12 +4025,15 @@ static void syncWriteForNumberOfTblInOneSql( uint64_t time_counter = winfo->start_time; int sampleUsePos; + int insert_interval = superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; int64_t st = 0; - int64_t et = 0; - for (int i = 0; i < superTblInfo->insertRows;) { + int64_t et = 0xffffffff; + + int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; + for (int i = 0; i < insertRows;) { int32_t tbl_id = 0; - for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; ) { - int64_t tmp_time = 0; + for (int tableSeq = winfo->start_table_id; tableSeq <= winfo->end_table_id; ) { + int64_t start_time = 0; int inserted = i; for (int k = 0; k < g_args.num_of_RPR;) { @@ -3879,12 +4041,12 @@ static void syncWriteForNumberOfTblInOneSql( memset(buffer, 0, superTblInfo->maxSqlLen); char *pstr = buffer; - int32_t end_tbl_id = tID + numberOfTblInOneSql; + int32_t end_tbl_id = tableSeq + numberOfTblInOneSql; if (end_tbl_id > winfo->end_table_id) { end_tbl_id = winfo->end_table_id+1; } - for (tbl_id = tID; tbl_id < end_tbl_id; tbl_id++) { + for (tbl_id = tableSeq ; tbl_id < end_tbl_id; tbl_id++) { sampleUsePos = samplePos; if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { char* tagsValBuf = NULL; @@ -3952,18 +4114,16 @@ static void syncWriteForNumberOfTblInOneSql( } } - tmp_time = time_counter; - for (k = 0; k < superTblInfo->rowsPerTbl;) { + start_time = time_counter; + for (int j = 0; j < superTblInfo->rowsPerTbl;) { int retLen = 0; if (0 == strncasecmp(superTblInfo->dataSource, "sample", strlen("sample"))) { - retLen = getRowDataFromSample(pstr + len, - superTblInfo->maxSqlLen - len, - tmp_time += superTblInfo->timeStampStep, - superTblInfo, - &sampleUsePos, - fp, - sampleDataBuf); + retLen = getRowDataFromSample(pstr + len, + superTblInfo->maxSqlLen - len, + start_time += superTblInfo->timeStampStep, + superTblInfo, + &sampleUsePos); if (retLen < 0) { goto free_and_statistics; } @@ -3972,15 +4132,15 @@ static void syncWriteForNumberOfTblInOneSql( int rand_num = rand_tinyint() % 100; if (0 != superTblInfo->disorderRatio && rand_num < superTblInfo->disorderRatio) { - int64_t d = tmp_time - rand() % superTblInfo->disorderRange; - retLen = generateRowData(pstr + len, - superTblInfo->maxSqlLen - len, - d, + int64_t d = start_time - taosRandom() % superTblInfo->disorderRange; + retLen = generateRowData(pstr + len, + superTblInfo->maxSqlLen - len, + d, superTblInfo); } else { - retLen = generateRowData(pstr + len, - superTblInfo->maxSqlLen - len, - tmp_time += superTblInfo->timeStampStep, + retLen = generateRowData(pstr + len, + superTblInfo->maxSqlLen - len, + start_time += superTblInfo->timeStampStep, superTblInfo); } if (retLen < 0) { @@ -3989,12 +4149,12 @@ static void syncWriteForNumberOfTblInOneSql( } len += retLen; //inserted++; - k++; - totalRowsInserted++; + j++; + winfo->totalInsertRows++; if (inserted >= superTblInfo->insertRows || (superTblInfo->maxSqlLen - len) < (superTblInfo->lenOfOneRow + 128)) { - tID = tbl_id + 1; + tableSeq = tbl_id + 1; printf("config rowsPerTbl and numberOfTblInOneSql not match with max_sql_lenth, please reconfig![lenOfOneRow:%d]\n", superTblInfo->lenOfOneRow); goto send_to_server; @@ -4002,18 +4162,18 @@ static void syncWriteForNumberOfTblInOneSql( } } - tID = tbl_id; + tableSeq = tbl_id; inserted += superTblInfo->rowsPerTbl; send_to_server: - if (g_args.insert_interval && (g_args.insert_interval > (et - st))) { - int sleep_time = g_args.insert_interval - (et -st); - printf("sleep: %d ms specified by insert_interval\n", sleep_time); - taosMsleep(sleep_time); // ms - } + if (insert_interval) { + st = taosGetTimestampUs(); - if (g_args.insert_interval) { - st = taosGetTimestampMs(); + if (insert_interval > ((et - st)/1000)) { + int sleep_time = insert_interval - (et -st); + printf("sleep: %d ms insert interval\n", sleep_time); + taosMsleep(sleep_time); // ms + } } if (0 == strncasecmp(superTblInfo->insertMode, @@ -4047,7 +4207,7 @@ send_to_server: if (currentPrintTime - lastPrintTime > 30*1000) { printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n", winfo->threadID, - winfo->totalRowsInserted, + winfo->totalInsertRows, winfo->totalAffectedRows); lastPrintTime = currentPrintTime; } @@ -4064,19 +4224,19 @@ send_to_server: goto free_and_statistics; } } - if (g_args.insert_interval) { - et = taosGetTimestampMs(); + if (insert_interval) { + et = taosGetTimestampUs(); } break; } - if (tID > winfo->end_table_id) { + if (tableSeq > winfo->end_table_id) { if (0 == strncasecmp(superTblInfo->dataSource, "sample", strlen("sample"))) { samplePos = sampleUsePos; } i = inserted; - time_counter = tmp_time; + time_counter = start_time; } } @@ -4085,14 +4245,13 @@ send_to_server: free_and_statistics: tmfree(buffer); - winfo->totalRowsInserted = totalRowsInserted; - winfo->totalAffectedRows = totalAffectedRows; - printf("====thread[%d] completed total inserted rows: %"PRId64 ", affected rows: %"PRId64 "====\n", winfo->threadID, totalRowsInserted, totalAffectedRows); + printf("====thread[%d] completed total inserted rows: %"PRId64 ", affected rows: %"PRId64 "====\n", + winfo->threadID, winfo->totalInsertRows, winfo->totalAffectedRows); return; } int32_t generateData(char *res, char **data_type, - int num_of_cols, int64_t timestamp, int len_of_binary) { + int num_of_cols, int64_t timestamp, int lenOfBinary) { memset(res, 0, MAX_DATA_SIZE); char *pstr = res; pstr += sprintf(pstr, "(%" PRId64, timestamp); @@ -4124,16 +4283,16 @@ int32_t generateData(char *res, char **data_type, double t = rand_double(); pstr += sprintf(pstr, ", %20.8f", t); } else if (strcasecmp(data_type[i % c], "bool") == 0) { - bool b = rand() & 1; + bool b = taosRandom() & 1; pstr += sprintf(pstr, ", %s", b ? "true" : "false"); } else if (strcasecmp(data_type[i % c], "binary") == 0) { - char *s = malloc(len_of_binary); - rand_string(s, len_of_binary); + char *s = malloc(lenOfBinary); + rand_string(s, lenOfBinary); pstr += sprintf(pstr, ", \"%s\"", s); free(s); }else if (strcasecmp(data_type[i % c], "nchar") == 0) { - char *s = malloc(len_of_binary); - rand_string(s, len_of_binary); + char *s = malloc(lenOfBinary); + rand_string(s, lenOfBinary); pstr += sprintf(pstr, ", \"%s\"", s); free(s); } @@ -4149,143 +4308,8 @@ int32_t generateData(char *res, char **data_type, return (int32_t)(pstr - res); } -// sync insertion -/* - 1 thread: 100 tables * 2000 rows/s - 1 thread: 10 tables * 20000 rows/s - 6 thread: 300 tables * 2000 rows/s - - 2 taosinsertdata , 1 thread: 10 tables * 20000 rows/s -*/ -static void* syncWrite(void *sarg) { - - threadInfo *winfo = (threadInfo *)sarg; - - char buffer[BUFFER_SIZE] = "\0"; - char data[MAX_DATA_SIZE]; - char **data_type = g_args.datatype; - int len_of_binary = g_args.len_of_binary; - - int ncols_per_record = 1; // count first col ts - int i = 0; - while(g_args.datatype[i]) { - i ++; - ncols_per_record ++; - } - - srand((uint32_t)time(NULL)); - int64_t time_counter = winfo->start_time; - - uint64_t st = 0; - uint64_t et = 0; - - winfo->totalRowsInserted = 0; - winfo->totalAffectedRows = 0; - - for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; tID++) { - int64_t tmp_time = time_counter; - - for (int i = 0; i < g_args.num_of_DPT;) { - - int tblInserted = i; - - char *pstr = buffer; - pstr += sprintf(pstr, - "insert into %s.%s%d values ", - winfo->db_name, g_args.tb_prefix, tID); - int k; - for (k = 0; k < g_args.num_of_RPR;) { - int rand_num = rand() % 100; - int len = -1; - - if ((g_args.disorderRatio != 0) - && (rand_num < g_args.disorderRange)) { - - int64_t d = tmp_time - rand() % 1000000 + rand_num; - len = generateData(data, data_type, - ncols_per_record, d, len_of_binary); - } else { - len = generateData(data, data_type, - ncols_per_record, tmp_time += 1000, len_of_binary); - } - - //assert(len + pstr - buffer < BUFFER_SIZE); - if (len + pstr - buffer >= BUFFER_SIZE) { // too long - break; - } - - pstr += sprintf(pstr, " %s", data); - tblInserted++; - k++; - i++; - - if (tblInserted >= g_args.num_of_DPT) - break; - } - - winfo->totalRowsInserted += k; - /* puts(buffer); */ - int64_t startTs; - int64_t endTs; - startTs = taosGetTimestampUs(); - //queryDB(winfo->taos, buffer); - if (i > 0 && g_args.insert_interval - && (g_args.insert_interval > (et - st) )) { - int sleep_time = g_args.insert_interval - (et -st); - printf("sleep: %d ms specified by insert_interval\n", sleep_time); - taosMsleep(sleep_time); // ms - } - - if (g_args.insert_interval) { - st = taosGetTimestampMs(); - } - verbosePrint("%s() LN%d %s\n", __func__, __LINE__, buffer); - int affectedRows = queryDbExec(winfo->taos, buffer, 1); - - if (0 < affectedRows){ - endTs = taosGetTimestampUs(); - int64_t delay = endTs - startTs; - if (delay > winfo->maxDelay) - winfo->maxDelay = delay; - if (delay < winfo->minDelay) - winfo->minDelay = delay; - winfo->cntDelay++; - winfo->totalDelay += delay; - winfo->totalAffectedRows += affectedRows; - winfo->avgDelay = (double)winfo->totalDelay / winfo->cntDelay; - } else { - fprintf(stderr, "queryDbExec() buffer:\n%s\naffected rows is %d", buffer, affectedRows); - } - - verbosePrint("%s() LN%d: totalaffectedRows:%"PRId64" tblInserted=%d\n", __func__, __LINE__, winfo->totalAffectedRows, tblInserted); - if (g_args.insert_interval) { - et = taosGetTimestampMs(); - } - - if (tblInserted >= g_args.num_of_DPT) { - break; - } - } // num_of_DPT - } // tId - - printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n", - winfo->threadID, - winfo->totalRowsInserted, - winfo->totalAffectedRows); - - return NULL; -} - - -static void* syncWriteWithStb(void *sarg) { - uint64_t lastPrintTime = taosGetTimestampMs(); - - threadInfo *winfo = (threadInfo *)sarg; - SSuperTable* superTblInfo = winfo->superTblInfo; - - FILE *fp = NULL; +static int prepareSampleDataForSTable(SSuperTable *superTblInfo) { char* sampleDataBuf = NULL; - int samplePos = 0; // each thread read sample data from csv file if (0 == strncasecmp(superTblInfo->dataSource, @@ -4294,193 +4318,304 @@ static void* syncWriteWithStb(void *sarg) { sampleDataBuf = calloc( superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1); if (sampleDataBuf == NULL) { - printf("Failed to calloc %d Bytes, reason:%s\n", + fprintf(stderr, "Failed to calloc %d Bytes, reason:%s\n", superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, strerror(errno)); - return NULL; + return -1; } - fp = fopen(superTblInfo->sampleFile, "r"); - if (fp == NULL) { - printf("Failed to open sample file: %s, reason:%s\n", - superTblInfo->sampleFile, strerror(errno)); - tmfree(sampleDataBuf); - return NULL; - } - int ret = readSampleFromCsvFileToMem(fp, - superTblInfo, sampleDataBuf); + int ret = readSampleFromCsvFileToMem(superTblInfo); if (0 != ret) { tmfree(sampleDataBuf); - tmfclose(fp); - return NULL; + return -1; } } - if (superTblInfo->numberOfTblInOneSql > 0) { - syncWriteForNumberOfTblInOneSql(winfo, fp, sampleDataBuf); - tmfree(sampleDataBuf); - tmfclose(fp); - return NULL; - } + superTblInfo->sampleDataBuf = sampleDataBuf; - char* buffer = calloc(superTblInfo->maxSqlLen, 1); - if (NULL == buffer) { - printf("Failed to calloc %d Bytes, reason:%s\n", - superTblInfo->maxSqlLen, - strerror(errno)); - tmfree(sampleDataBuf); - tmfclose(fp); - return NULL; - } + return 0; +} - uint64_t st = 0; - uint64_t et = 0; +static int execInsert(threadInfo *winfo, char *buffer, int k) +{ + int affectedRows; + SSuperTable* superTblInfo = winfo->superTblInfo; - winfo->totalRowsInserted = 0; - winfo->totalAffectedRows = 0; + if (superTblInfo) { + if (0 == strncasecmp(superTblInfo->insertMode, "taosc", strlen("taosc"))) { + verbosePrint("%s() LN%d %s\n", __func__, __LINE__, buffer); + affectedRows = queryDbExec(winfo->taos, buffer, INSERT_TYPE); + } else { + verbosePrint("%s() LN%d %s\n", __func__, __LINE__, buffer); + int retCode = postProceSql(g_Dbs.host, g_Dbs.port, buffer); - int sampleUsePos; + if (0 != retCode) { + affectedRows = -1; + printf("========restful return fail, threadID[%d]\n", winfo->threadID); + } else { + affectedRows = k; + } + } + } else { + verbosePrint("%s() LN%d %s\n", __func__, __LINE__, buffer); + affectedRows = queryDbExec(winfo->taos, buffer, 1); + } - verbosePrint("%s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, superTblInfo->insertRows); + if (0 > affectedRows){ + return affectedRows; + } - for (uint32_t tID = winfo->start_table_id; tID <= winfo->end_table_id; - tID++) { - int64_t start_time = winfo->start_time; + return affectedRows; +} - for (int i = 0; i < superTblInfo->insertRows;) { +static int generateDataBuffer(int32_t tableSeq, + threadInfo *pThreadInfo, char *buffer, + int64_t insertRows, + int64_t startFrom, int64_t startTime, int *pSampleUsePos) +{ + SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - int64_t tblInserted = i; + int ncols_per_record = 1; // count first col ts - if (i > 0 && g_args.insert_interval - && (g_args.insert_interval > (et - st) )) { - int sleep_time = g_args.insert_interval - (et -st); - printf("sleep: %d ms specified by insert_interval\n", sleep_time); - taosMsleep(sleep_time); // ms - } + if (superTblInfo == NULL) { + int datatypeSeq = 0; + while(g_args.datatype[datatypeSeq]) { + datatypeSeq ++; + ncols_per_record ++; + } + } - if (g_args.insert_interval) { - st = taosGetTimestampMs(); - } + assert(buffer != NULL); - sampleUsePos = samplePos; - verbosePrint("%s() LN%d num_of_RPR=%d\n", __func__, __LINE__, g_args.num_of_RPR); + char *pChildTblName; + int childTblCount; - memset(buffer, 0, superTblInfo->maxSqlLen); - int len = 0; + if (superTblInfo && (superTblInfo->childTblOffset > 0)) { + // select tbname from stb limit 1 offset tableSeq + getChildNameOfSuperTableWithLimitAndOffset(pThreadInfo->taos, + pThreadInfo->db_name, superTblInfo->sTblName, + &pChildTblName, &childTblCount, + 1, tableSeq); + } else { + pChildTblName = calloc(TSDB_TABLE_NAME_LEN, 1); + if (NULL == pChildTblName) { + fprintf(stderr, "failed to alloc memory %d\n", TSDB_TABLE_NAME_LEN); + return -1; + } + snprintf(pChildTblName, TSDB_TABLE_NAME_LEN, "%s%d", + superTblInfo?superTblInfo->childTblPrefix:g_args.tb_prefix, tableSeq); + } - char *pstr = buffer; + memset(buffer, 0, superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len); - if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { - char* tagsValBuf = NULL; - if (0 == superTblInfo->tagSource) { + char *pstr = buffer; + + if (superTblInfo) { + if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { + char* tagsValBuf = NULL; + if (0 == superTblInfo->tagSource) { tagsValBuf = generateTagVaulesForStb(superTblInfo); - } else { - tagsValBuf = getTagValueFromTagSample( - superTblInfo, - tID % superTblInfo->tagSampleCount); - } - if (NULL == tagsValBuf) { - goto free_and_statistics_2; - } - - len += snprintf(pstr + len, - superTblInfo->maxSqlLen - len, - "insert into %s.%s%d using %s.%s tags %s values", - winfo->db_name, - superTblInfo->childTblPrefix, - tID, - winfo->db_name, - superTblInfo->sTblName, - tagsValBuf); - tmfree(tagsValBuf); - } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) { - len += snprintf(pstr + len, - superTblInfo->maxSqlLen - len, - "insert into %s.%s values", - winfo->db_name, - superTblInfo->childTblName + tID * TSDB_TABLE_NAME_LEN); } else { - len += snprintf(pstr + len, - superTblInfo->maxSqlLen - len, - "insert into %s.%s%d values", - winfo->db_name, - superTblInfo->childTblPrefix, - tID); + tagsValBuf = getTagValueFromTagSample( + superTblInfo, + tableSeq % superTblInfo->tagSampleCount); + } + if (NULL == tagsValBuf) { + fprintf(stderr, "tag buf failed to allocate memory\n"); + free(pChildTblName); + return -1; } - int k; - for (k = 0; k < g_args.num_of_RPR;) { - int retLen = 0; - if (0 == strncasecmp(superTblInfo->dataSource, "sample", strlen("sample"))) { - retLen = getRowDataFromSample( + pstr += snprintf(pstr, + superTblInfo->maxSqlLen, + "insert into %s.%s using %s.%s tags %s values", + pThreadInfo->db_name, + pChildTblName, + pThreadInfo->db_name, + superTblInfo->sTblName, + tagsValBuf); + tmfree(tagsValBuf); + } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) { + pstr += snprintf(pstr, + superTblInfo->maxSqlLen, + "insert into %s.%s values", + pThreadInfo->db_name, + superTblInfo->childTblName + tableSeq * TSDB_TABLE_NAME_LEN); + } else { + pstr += snprintf(pstr, + (superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len), + "insert into %s.%s values", + pThreadInfo->db_name, + pChildTblName); + } + } else { + pstr += snprintf(pstr, + g_args.max_sql_len, + "insert into %s.%s values", + pThreadInfo->db_name, + pChildTblName); + } + + int k; + int len = 0; + + verbosePrint("%s() LN%d num_of_RPR=%d\n", __func__, __LINE__, g_args.num_of_RPR); + for (k = 0; k < g_args.num_of_RPR;) { + if (superTblInfo) { + int retLen = 0; + + if (0 == strncasecmp(superTblInfo->dataSource, + "sample", strlen("sample"))) { + retLen = getRowDataFromSample( pstr + len, superTblInfo->maxSqlLen - len, - start_time + superTblInfo->timeStampStep * i, + startTime + superTblInfo->timeStampStep * startFrom, superTblInfo, - &sampleUsePos, - fp, - sampleDataBuf); - if (retLen < 0) { - goto free_and_statistics_2; - } - } else if (0 == strncasecmp(superTblInfo->dataSource, "rand", strlen("rand"))) { - int rand_num = rand_tinyint() % 100; - if (0 != superTblInfo->disorderRatio + pSampleUsePos); + } else if (0 == strncasecmp(superTblInfo->dataSource, + "rand", strlen("rand"))) { + int rand_num = rand_tinyint() % 100; + if (0 != superTblInfo->disorderRatio && rand_num < superTblInfo->disorderRatio) { - int64_t d = start_time - rand() % superTblInfo->disorderRange; - retLen = generateRowData( + int64_t d = startTime - taosRandom() % superTblInfo->disorderRange; + retLen = generateRowData( pstr + len, superTblInfo->maxSqlLen - len, d, superTblInfo); - //printf("disorder rows, rand_num:%d, last ts:%"PRId64" current ts:%"PRId64"\n", rand_num, tmp_time, d); + //printf("disorder rows, rand_num:%d, last ts:%"PRId64" current ts:%"PRId64"\n", rand_num, start_time, d); } else { - retLen = generateRowData( + retLen = generateRowData( pstr + len, superTblInfo->maxSqlLen - len, - start_time + superTblInfo->timeStampStep * i, + startTime + superTblInfo->timeStampStep * startFrom, superTblInfo); - } - if (retLen < 0) { - goto free_and_statistics_2; - } + } + + if (retLen < 0) { + free(pChildTblName); + return -1; } len += retLen; - verbosePrint("%s() LN%d retLen=%d len=%d k=%d \nbuffer=%s\n", __func__, __LINE__, retLen, len, k, buffer); + } + } else { + int rand_num = taosRandom() % 100; + char data[MAX_DATA_SIZE]; + char **data_type = g_args.datatype; + int lenOfBinary = g_args.len_of_binary; - tblInserted++; - k++; - i++; + if ((g_args.disorderRatio != 0) + && (rand_num < g_args.disorderRange)) { + + int64_t d = startTime - taosRandom() % 1000000 + rand_num; + len = generateData(data, data_type, + ncols_per_record, d, lenOfBinary); + } else { + len = generateData(data, data_type, + ncols_per_record, + startTime + DEFAULT_TIMESTAMP_STEP * startFrom, + lenOfBinary); + } - if (tblInserted >= superTblInfo->insertRows) + //assert(len + pstr - buffer < BUFFER_SIZE); + if (len + pstr - buffer >= g_args.max_sql_len) { // too long break; } - - winfo->totalRowsInserted += k; - int64_t startTs = taosGetTimestampUs(); - int64_t endTs; - int affectedRows; - if (0 == strncasecmp(superTblInfo->insertMode, "taosc", strlen("taosc"))) { - verbosePrint("%s() LN%d %s\n", __func__, __LINE__, buffer); - affectedRows = queryDbExec(winfo->taos, buffer, INSERT_TYPE); + pstr += sprintf(pstr, " %s", data); + } - if (0 > affectedRows){ - goto free_and_statistics_2; - } - } else { - verbosePrint("%s() LN%d %s\n", __func__, __LINE__, buffer); - int retCode = postProceSql(g_Dbs.host, g_Dbs.port, buffer); + verbosePrint("%s() LN%d len=%d k=%d \nbuffer=%s\n", __func__, __LINE__, len, k, buffer); - if (0 != retCode) { - printf("========restful return fail, threadID[%d]\n", winfo->threadID); - goto free_and_statistics_2; - } + k++; + startFrom ++; - affectedRows = k; + if (startFrom >= insertRows) + break; + } + + free(pChildTblName); + + return k; +} + +// sync insertion +/* + 1 thread: 100 tables * 2000 rows/s + 1 thread: 10 tables * 20000 rows/s + 6 thread: 300 tables * 2000 rows/s + + 2 taosinsertdata , 1 thread: 10 tables * 20000 rows/s +*/ +static void* syncWrite(void *sarg) { + + threadInfo *winfo = (threadInfo *)sarg; + SSuperTable* superTblInfo = winfo->superTblInfo; + + char* buffer = calloc(superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len, 1); + if (NULL == buffer) { + fprintf(stderr, "Failed to alloc %d Bytes, reason:%s\n", + superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len, + strerror(errno)); + return NULL; + } + + if (superTblInfo) { + if (0 != prepareSampleDataForSTable(superTblInfo)) + return NULL; + + if (superTblInfo->numberOfTblInOneSql > 0) { + syncWriteForNumberOfTblInOneSql(winfo, superTblInfo->sampleDataBuf); + tmfree(superTblInfo->sampleDataBuf); + return NULL; + } + } + + int samplePos = 0; + + int64_t lastPrintTime = taosGetTimestampMs(); + int64_t startTs = taosGetTimestampUs(); + int64_t endTs; + + int insert_interval = superTblInfo?superTblInfo->insertInterval: + g_args.insert_interval; + uint64_t st = 0; + uint64_t et = 0xffffffff; + + winfo->totalInsertRows = 0; + winfo->totalAffectedRows = 0; + + int sampleUsePos; + + for (uint32_t tableSeq = winfo->start_table_id; tableSeq <= winfo->end_table_id; + tableSeq ++) { + int64_t start_time = winfo->start_time; + + int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; + verbosePrint("%s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, insertRows); + + for (int64_t i = 0; i < insertRows;) { + if (insert_interval) { + st = taosGetTimestampUs(); } + sampleUsePos = samplePos; + + int generated = generateDataBuffer(tableSeq, winfo, buffer, insertRows, + i, start_time, &sampleUsePos); + if (generated > 0) + i += generated; + else + goto free_and_statistics_2; + + int affectedRows = execInsert(winfo, buffer, generated); + if (affectedRows < 0) + goto free_and_statistics_2; + + winfo->totalInsertRows += generated; + winfo->totalAffectedRows += affectedRows; + endTs = taosGetTimestampUs(); int64_t delay = endTs - startTs; if (delay > winfo->maxDelay) winfo->maxDelay = delay; @@ -4488,54 +4623,57 @@ static void* syncWriteWithStb(void *sarg) { winfo->cntDelay++; winfo->totalDelay += delay; - winfo->totalAffectedRows += affectedRows; - int64_t currentPrintTime = taosGetTimestampMs(); if (currentPrintTime - lastPrintTime > 30*1000) { printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n", winfo->threadID, - winfo->totalRowsInserted, + winfo->totalInsertRows, winfo->totalAffectedRows); lastPrintTime = currentPrintTime; } - if (g_args.insert_interval) { - et = taosGetTimestampMs(); - } - - if (tblInserted >= superTblInfo->insertRows) + if (i >= insertRows) break; + + if (insert_interval) { + et = taosGetTimestampUs(); + + if (insert_interval > ((et - st)/1000) ) { + int sleep_time = insert_interval - (et -st)/1000; + verbosePrint("%s() LN%d sleep: %d ms for insert interval\n", __func__, __LINE__, sleep_time); + taosMsleep(sleep_time); // ms + } + } } // num_of_DPT - if (tID == winfo->end_table_id) { - if (0 == strncasecmp( - superTblInfo->dataSource, "sample", strlen("sample"))) { + if ((tableSeq == winfo->end_table_id) && superTblInfo && + (0 == strncasecmp( + superTblInfo->dataSource, "sample", strlen("sample")))) { samplePos = sampleUsePos; - } - } - //printf("========loop %d childTables duration:%"PRId64 "========inserted rows:%d\n", winfo->end_table_id - winfo->start_table_id, et - st, i); - } // tID + } // tableSeq free_and_statistics_2: tmfree(buffer); - tmfree(sampleDataBuf); - tmfclose(fp); + if (superTblInfo) + tmfree(superTblInfo->sampleDataBuf); printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n", winfo->threadID, - winfo->totalRowsInserted, + winfo->totalInsertRows, winfo->totalAffectedRows); return NULL; } void callBack(void *param, TAOS_RES *res, int code) { threadInfo* winfo = (threadInfo*)param; + SSuperTable* superTblInfo = winfo->superTblInfo; - if (g_args.insert_interval) { - winfo->et = taosGetTimestampMs(); - if (winfo->et - winfo->st < 1000) { - taosMsleep(1000 - (winfo->et - winfo->st)); // ms + int insert_interval = superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; + if (insert_interval) { + winfo->et = taosGetTimestampUs(); + if (((winfo->et - winfo->st)/1000) < insert_interval) { + taosMsleep(insert_interval - (winfo->et - winfo->st)/1000); // ms } } @@ -4557,14 +4695,14 @@ void callBack(void *param, TAOS_RES *res, int code) { } for (int i = 0; i < g_args.num_of_RPR; i++) { - int rand_num = rand() % 100; + int rand_num = taosRandom() % 100; if (0 != winfo->superTblInfo->disorderRatio && rand_num < winfo->superTblInfo->disorderRatio) { - int64_t d = winfo->lastTs - rand() % 1000000 + rand_num; + int64_t d = winfo->lastTs - taosRandom() % 1000000 + rand_num; //generateData(data, datatype, ncols_per_record, d, len_of_binary); (void)generateRowData(data, MAX_DATA_SIZE, d, winfo->superTblInfo); } else { - //generateData(data, datatype, ncols_per_record, tmp_time += 1000, len_of_binary); + //generateData(data, datatype, ncols_per_record, start_time += 1000, len_of_binary); (void)generateRowData(data, MAX_DATA_SIZE, winfo->lastTs += 1000, winfo->superTblInfo); } pstr += sprintf(pstr, "%s", data); @@ -4575,8 +4713,8 @@ void callBack(void *param, TAOS_RES *res, int code) { } } - if (g_args.insert_interval) { - winfo->st = taosGetTimestampMs(); + if (insert_interval) { + winfo->st = taosGetTimestampUs(); } taos_query_a(winfo->taos, buffer, callBack, winfo); free(buffer); @@ -4587,13 +4725,15 @@ void callBack(void *param, TAOS_RES *res, int code) { void *asyncWrite(void *sarg) { threadInfo *winfo = (threadInfo *)sarg; + SSuperTable* superTblInfo = winfo->superTblInfo; winfo->st = 0; winfo->et = 0; winfo->lastTs = winfo->start_time; - if (g_args.insert_interval) { - winfo->st = taosGetTimestampMs(); + int insert_interval = superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; + if (insert_interval) { + winfo->st = taosGetTimestampUs(); } taos_query_a(winfo->taos, "show databases", callBack, winfo); @@ -4602,17 +4742,24 @@ void *asyncWrite(void *sarg) { return NULL; } -void startMultiThreadInsertData(int threads, char* db_name, char* precision, - SSuperTable* superTblInfo) { +static void startMultiThreadInsertData(int threads, char* db_name, + char* precision,SSuperTable* superTblInfo) { pthread_t *pids = malloc(threads * sizeof(pthread_t)); + assert(pids != NULL); + threadInfo *infos = malloc(threads * sizeof(threadInfo)); + assert(infos != NULL); + memset(pids, 0, threads * sizeof(pthread_t)); memset(infos, 0, threads * sizeof(threadInfo)); int ntables = 0; if (superTblInfo) - ntables = superTblInfo->childTblCount; + if (superTblInfo->childTblOffset) + ntables = superTblInfo->childTblLimit; + else + ntables = superTblInfo->childTblCount; else ntables = g_args.num_of_tables; @@ -4643,7 +4790,7 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, } else if (0 == strncasecmp(precision, "us", 2)) { timePrec = TSDB_TIME_PRECISION_MICRO; } else { - printf("No support precision: %s\n", precision); + fprintf(stderr, "No support precision: %s\n", precision); exit(-1); } } @@ -4652,11 +4799,11 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, if (superTblInfo) { if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { start_time = taosGetTimestamp(timePrec); - } else { + } else { if (TSDB_CODE_SUCCESS != taosParseTime( - superTblInfo->startTimestamp, - &start_time, - strlen(superTblInfo->startTimestamp), + superTblInfo->startTimestamp, + &start_time, + strlen(superTblInfo->startTimestamp), timePrec, 0)) { printf("ERROR to parse time!\n"); exit(-1); @@ -4667,8 +4814,14 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, } double start = getCurrentTime(); - - int last = 0; + + int last; + + if ((superTblInfo) && (superTblInfo->childTblOffset)) + last = superTblInfo->childTblOffset; + else + last = 0; + for (int i = 0; i < threads; i++) { threadInfo *t_info = infos + i; t_info->threadID = i; @@ -4682,7 +4835,7 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5))) { //t_info->taos = taos; t_info->taos = taos_connect( - g_Dbs.host, g_Dbs.user, + g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); if (NULL == t_info->taos) { printf("connect to server fail from insert sub thread, reason: %s\n", @@ -4706,16 +4859,12 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, tsem_init(&(t_info->lock_sem), 0, 0); if (SYNC == g_Dbs.queryMode) { - if (superTblInfo) { - pthread_create(pids + i, NULL, syncWriteWithStb, t_info); - } else { - pthread_create(pids + i, NULL, syncWrite, t_info); - } - } else { + pthread_create(pids + i, NULL, syncWrite, t_info); + } else { pthread_create(pids + i, NULL, asyncWrite, t_info); } } - + for (int i = 0; i < threads; i++) { pthread_join(pids[i], NULL); } @@ -4734,13 +4883,13 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, if (superTblInfo) { superTblInfo->totalAffectedRows += t_info->totalAffectedRows; - superTblInfo->totalRowsInserted += t_info->totalRowsInserted; + superTblInfo->totalInsertRows += t_info->totalInsertRows; } totalDelay += t_info->totalDelay; cntDelay += t_info->cntDelay; if (t_info->maxDelay > maxDelay) maxDelay = t_info->maxDelay; - if (t_info->minDelay < minDelay) minDelay = t_info->minDelay; + if (t_info->minDelay < minDelay) minDelay = t_info->minDelay; } cntDelay -= 1; @@ -4751,16 +4900,17 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, double t = end - start; if (superTblInfo) { - printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s. %2.f records/second\n\n", - t, superTblInfo->totalRowsInserted, + printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s. %2.f records/second\n\n", + t, superTblInfo->totalInsertRows, superTblInfo->totalAffectedRows, threads, db_name, superTblInfo->sTblName, - superTblInfo->totalRowsInserted / t); - fprintf(g_fpOfInsertResult, "Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s. %2.f records/second\n\n", - t, superTblInfo->totalRowsInserted, + superTblInfo->totalInsertRows / t); + fprintf(g_fpOfInsertResult, + "Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s. %2.f records/second\n\n", + t, superTblInfo->totalInsertRows, superTblInfo->totalAffectedRows, threads, db_name, superTblInfo->sTblName, - superTblInfo->totalRowsInserted / t); + superTblInfo->totalInsertRows/ t); } printf("insert delay, avg: %10.6fms, max: %10.6fms, min: %10.6fms\n\n", @@ -4771,10 +4921,9 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, //taos_close(taos); free(pids); - free(infos); + free(infos); } - void *readTable(void *sarg) { #if 1 threadInfo *rinfo = (threadInfo *)sarg; @@ -4919,7 +5068,7 @@ void *readMetric(void *sarg) { } -int insertTestProcess() { +static int insertTestProcess() { setupForAnsiEscape(); int ret = printfInsertMeta(); @@ -4933,15 +5082,15 @@ int insertTestProcess() { if (NULL == g_fpOfInsertResult) { fprintf(stderr, "Failed to open %s for save result\n", g_Dbs.resultFile); return -1; - } { - printfInsertMetaToFile(g_fpOfInsertResult); } + printfInsertMetaToFile(g_fpOfInsertResult); + if (!g_args.answer_yes) { printf("Press enter key to continue\n\n"); (void)getchar(); } - + init_rand_data(); // create database and super tables @@ -4951,7 +5100,7 @@ int insertTestProcess() { } // pretreatement - prePareSampleData(); + prepareSampleData(); double start; double end; @@ -4962,47 +5111,47 @@ int insertTestProcess() { end = getCurrentTime(); if (g_totalChildTables > 0) { - printf("Spent %.4f seconds to create %d tables with %d thread(s)\n\n", + printf("Spent %.4f seconds to create %d tables with %d thread(s)\n\n", end - start, g_totalChildTables, g_Dbs.threadCount); - fprintf(g_fpOfInsertResult, - "Spent %.4f seconds to create %d tables with %d thread(s)\n\n", + fprintf(g_fpOfInsertResult, + "Spent %.4f seconds to create %d tables with %d thread(s)\n\n", end - start, g_totalChildTables, g_Dbs.threadCount); } taosMsleep(1000); // create sub threads for inserting data //start = getCurrentTime(); - for (int i = 0; i < g_Dbs.dbCount; i++) { - if (g_Dbs.db[i].superTblCount > 0) { - for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { - SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j]; - if (0 == g_Dbs.db[i].superTbls[j].insertRows) { - continue; - } - startMultiThreadInsertData( - g_Dbs.threadCount, - g_Dbs.db[i].dbName, - g_Dbs.db[i].dbCfg.precision, - superTblInfo); - } - } else { - startMultiThreadInsertData( - g_Dbs.threadCount, - g_Dbs.db[i].dbName, - g_Dbs.db[i].dbCfg.precision, - NULL); + for (int i = 0; i < g_Dbs.dbCount; i++) { + if (g_Dbs.db[i].superTblCount > 0) { + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j]; + if (0 == g_Dbs.db[i].superTbls[j].insertRows) { + continue; } + startMultiThreadInsertData( + g_Dbs.threadCount, + g_Dbs.db[i].dbName, + g_Dbs.db[i].dbCfg.precision, + superTblInfo); + } + } else { + startMultiThreadInsertData( + g_Dbs.threadCount, + g_Dbs.db[i].dbName, + g_Dbs.db[i].dbCfg.precision, + NULL); } + } //end = getCurrentTime(); - //int64_t totalRowsInserted = 0; + //int64_t totalInsertRows = 0; //int64_t totalAffectedRows = 0; //for (int i = 0; i < g_Dbs.dbCount; i++) { // for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { - // totalRowsInserted += g_Dbs.db[i].superTbls[j].totalRowsInserted; + // totalInsertRows+= g_Dbs.db[i].superTbls[j].totalInsertRows; // totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows; //} - //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalRowsInserted, totalAffectedRows, g_Dbs.threadCount); + //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalInsertRows, totalAffectedRows, g_Dbs.threadCount); postFreeResource(); return 0; @@ -5023,7 +5172,7 @@ void *superQueryProcess(void *sarg) { //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_id, winfo->end_table_id); } - st = taosGetTimestampMs(); + st = taosGetTimestampUs(); for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { int64_t t1 = taosGetTimestampUs(); @@ -5049,14 +5198,14 @@ void *superQueryProcess(void *sarg) { } } } - et = taosGetTimestampMs(); + et = taosGetTimestampUs(); printf("==thread[%"PRId64"] complete all sqls to specify tables once queries duration:%.6fs\n\n", taosGetSelfPthreadId(), (double)(et - st)/1000.0); } return NULL; } -void replaceSubTblName(char* inSql, char* outSql, int tblIndex) { +static void replaceSubTblName(char* inSql, char* outSql, int tblIndex) { char sourceString[32] = "xxxx"; char subTblName[MAX_TB_NAME_SIZE*3]; sprintf(subTblName, "%s.%s", @@ -5078,7 +5227,7 @@ void replaceSubTblName(char* inSql, char* outSql, int tblIndex) { //printf("3: %s\n", outSql); } -void *subQueryProcess(void *sarg) { +static void *subQueryProcess(void *sarg) { char sqlstr[1024]; threadInfo *winfo = (threadInfo *)sarg; int64_t st = 0; @@ -5089,7 +5238,7 @@ void *subQueryProcess(void *sarg) { //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_id, winfo->end_table_id); } - st = taosGetTimestampMs(); + st = taosGetTimestampUs(); for (int i = winfo->start_table_id; i <= winfo->end_table_id; i++) { for (int j = 0; j < g_queryInfo.subQueryInfo.sqlCount; j++) { memset(sqlstr,0,sizeof(sqlstr)); @@ -5103,12 +5252,12 @@ void *subQueryProcess(void *sarg) { selectAndGetResult(winfo->taos, sqlstr, tmpFile); } } - et = taosGetTimestampMs(); + et = taosGetTimestampUs(); printf("####thread[%"PRId64"] complete all sqls to allocate all sub-tables[%d - %d] once queries duration:%.4fs\n\n", taosGetSelfPthreadId(), winfo->start_table_id, winfo->end_table_id, - (double)(et - st)/1000.0); + (double)(et - st)/1000000.0); } return NULL; } @@ -5126,10 +5275,10 @@ static int queryTestProcess() { } if (0 != g_queryInfo.subQueryInfo.sqlCount) { - (void)getAllChildNameOfSuperTable(taos, - g_queryInfo.dbName, - g_queryInfo.subQueryInfo.sTblName, - &g_queryInfo.subQueryInfo.childTblName, + getAllChildNameOfSuperTable(taos, + g_queryInfo.dbName, + g_queryInfo.subQueryInfo.sTblName, + &g_queryInfo.subQueryInfo.childTblName, &g_queryInfo.subQueryInfo.childTblCount); } @@ -5146,7 +5295,7 @@ static int queryTestProcess() { threadInfo *infos = NULL; //==== create sub threads for query from specify table if (g_queryInfo.superQueryInfo.sqlCount > 0 && g_queryInfo.superQueryInfo.concurrent > 0) { - + pids = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(pthread_t)); infos = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(threadInfo)); if ((NULL == pids) || (NULL == infos)) { @@ -5154,14 +5303,14 @@ static int queryTestProcess() { taos_close(taos); exit(-1); } - - for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { + + for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { threadInfo *t_info = infos + i; - t_info->threadID = i; + t_info->threadID = i; if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { t_info->taos = taos; - + char sqlStr[MAX_TB_NAME_SIZE*2]; sprintf(sqlStr, "use %s", g_queryInfo.dbName); verbosePrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr); @@ -5170,16 +5319,17 @@ static int queryTestProcess() { t_info->taos = NULL; } - pthread_create(pids + i, NULL, superQueryProcess, t_info); - } + pthread_create(pids + i, NULL, superQueryProcess, t_info); + } }else { g_queryInfo.superQueryInfo.concurrent = 0; } - + pthread_t *pidsOfSub = NULL; threadInfo *infosOfSub = NULL; //==== create sub threads for query from all sub table of the super table - if ((g_queryInfo.subQueryInfo.sqlCount > 0) && (g_queryInfo.subQueryInfo.threadCnt > 0)) { + if ((g_queryInfo.subQueryInfo.sqlCount > 0) + && (g_queryInfo.subQueryInfo.threadCnt > 0)) { pidsOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(pthread_t)); infosOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(threadInfo)); if ((NULL == pidsOfSub) || (NULL == infosOfSub)) { @@ -5187,7 +5337,7 @@ static int queryTestProcess() { taos_close(taos); exit(-1); } - + int ntables = g_queryInfo.subQueryInfo.childTblCount; int threads = g_queryInfo.subQueryInfo.threadCnt; @@ -5196,12 +5346,12 @@ static int queryTestProcess() { threads = ntables; a = 1; } - + int b = 0; if (threads != 0) { b = ntables % threads; } - + int last = 0; for (int i = 0; i < threads; i++) { threadInfo *t_info = infosOfSub + i; @@ -5395,7 +5545,7 @@ void *superSubscribeProcess(void *sarg) { } } taos_free_result(res); - + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { taos_unsubscribe(g_queryInfo.superQueryInfo.tsub[i], g_queryInfo.superQueryInfo.subscribeKeepProgress); @@ -5423,14 +5573,13 @@ static int subscribeTestProcess() { } if (0 != g_queryInfo.subQueryInfo.sqlCount) { - (void)getAllChildNameOfSuperTable(taos, + getAllChildNameOfSuperTable(taos, g_queryInfo.dbName, g_queryInfo.subQueryInfo.sTblName, &g_queryInfo.subQueryInfo.childTblName, &g_queryInfo.subQueryInfo.childTblCount); } - pthread_t *pids = NULL; threadInfo *infos = NULL; //==== create sub threads for query from super table @@ -5439,53 +5588,53 @@ static int subscribeTestProcess() { pids = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(pthread_t)); infos = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(threadInfo)); if ((NULL == pids) || (NULL == infos)) { - printf("malloc failed for create threads\n"); + printf("malloc failed for create threads\n"); taos_close(taos); exit(-1); } - - for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { + + for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { threadInfo *t_info = infos + i; t_info->threadID = i; t_info->taos = taos; pthread_create(pids + i, NULL, superSubscribeProcess, t_info); } } - + //==== create sub threads for query from sub table pthread_t *pidsOfSub = NULL; threadInfo *infosOfSub = NULL; - if ((g_queryInfo.subQueryInfo.sqlCount > 0) + if ((g_queryInfo.subQueryInfo.sqlCount > 0) && (g_queryInfo.subQueryInfo.threadCnt > 0)) { - pidsOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * + pidsOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(pthread_t)); infosOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(threadInfo)); if ((NULL == pidsOfSub) || (NULL == infosOfSub)) { - printf("malloc failed for create threads\n"); + printf("malloc failed for create threads\n"); taos_close(taos); exit(-1); } - + int ntables = g_queryInfo.subQueryInfo.childTblCount; int threads = g_queryInfo.subQueryInfo.threadCnt; - + int a = ntables / threads; if (a < 1) { threads = ntables; a = 1; } - + int b = 0; if (threads != 0) { b = ntables % threads; } - + int last = 0; - for (int i = 0; i < threads; i++) { + for (int i = 0; i < threads; i++) { threadInfo *t_info = infosOfSub + i; t_info->threadID = i; - + t_info->start_table_id = last; t_info->end_table_id = i < b ? last + a : last + a - 1; t_info->taos = taos; @@ -5493,20 +5642,20 @@ static int subscribeTestProcess() { } g_queryInfo.subQueryInfo.threadCnt = threads; } - + for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { pthread_join(pids[i], NULL); } tmfree((char*)pids); - tmfree((char*)infos); + tmfree((char*)infos); for (int i = 0; i < g_queryInfo.subQueryInfo.threadCnt; i++) { pthread_join(pidsOfSub[i], NULL); } tmfree((char*)pidsOfSub); - tmfree((char*)infosOfSub); + tmfree((char*)infosOfSub); taos_close(taos); return 0; } @@ -5599,9 +5748,9 @@ void setParaFromArg(){ tstrncpy(g_Dbs.db[0].superTbls[0].insertMode, "taosc", MAX_TB_NAME_SIZE); tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp, "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE); - g_Dbs.db[0].superTbls[0].timeStampStep = 10; + g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP; - g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT; + g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT; g_Dbs.db[0].superTbls[0].maxSqlLen = TSDB_PAYLOAD_SIZE; g_Dbs.db[0].superTbls[0].columnCount = 0; @@ -5745,14 +5894,7 @@ static void testMetaFile() { } } -static void testCmdLine() { - - g_args.test_mode = INSERT_MODE; - insertTestProcess(); - - if (g_Dbs.insert_only) - return; - +static void queryResult() { // select if (false == g_Dbs.insert_only) { // query data @@ -5798,6 +5940,17 @@ static void testCmdLine() { } } +static void testCmdLine() { + + g_args.test_mode = INSERT_MODE; + insertTestProcess(); + + if (g_Dbs.insert_only) + return; + else + queryResult(); +} + int main(int argc, char *argv[]) { parse_args(argc, argv, &g_args); diff --git a/src/os/inc/osTime.h b/src/os/inc/osTime.h index b20ccadadb22a04733d97bd19b919660ee677d0d..2c50e7eeabd934a9d88f32d4b3bdc29d41d699b1 100644 --- a/src/os/inc/osTime.h +++ b/src/os/inc/osTime.h @@ -72,6 +72,11 @@ typedef struct SInterval { int64_t offset; } SInterval; +typedef struct SSessionWindow { + int64_t gap; // gap between two session window(in microseconds) + int32_t primaryColId; // primary timestamp column +} SSessionWindow; + int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision); int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision); int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision); diff --git a/src/plugins/http/inc/httpJson.h b/src/plugins/http/inc/httpJson.h index fcb74253b99c0cd8921d981e3ba7a4a027bf2b31..4d182d0132528367a8e25b8f72329331be2cb5c5 100644 --- a/src/plugins/http/inc/httpJson.h +++ b/src/plugins/http/inc/httpJson.h @@ -63,9 +63,11 @@ void httpJsonString(JsonBuf* buf, char* sVal, int32_t len); void httpJsonOriginString(JsonBuf* buf, char* sVal, int32_t len); void httpJsonStringForTransMean(JsonBuf* buf, char* SVal, int32_t maxLen); void httpJsonInt64(JsonBuf* buf, int64_t num); +void httpJsonUInt64(JsonBuf* buf, uint64_t num); void httpJsonTimestamp(JsonBuf* buf, int64_t t, bool us); void httpJsonUtcTimestamp(JsonBuf* buf, int64_t t, bool us); void httpJsonInt(JsonBuf* buf, int32_t num); +void httpJsonUInt(JsonBuf* buf, uint32_t num); void httpJsonFloat(JsonBuf* buf, float num); void httpJsonDouble(JsonBuf* buf, double num); void httpJsonNull(JsonBuf* buf); diff --git a/src/plugins/http/src/httpJson.c b/src/plugins/http/src/httpJson.c index b120496898c12e4f189337387a5f9fad1fadccbe..19166e720f0e517ccd1a793742836ae4bc245bd1 100644 --- a/src/plugins/http/src/httpJson.c +++ b/src/plugins/http/src/httpJson.c @@ -256,6 +256,12 @@ void httpJsonInt64(JsonBuf* buf, int64_t num) { buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%" PRId64, num); } +void httpJsonUInt64(JsonBuf* buf, uint64_t num) { + httpJsonItemToken(buf); + httpJsonTestBuf(buf, MAX_NUM_STR_SZ); + buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%" PRIu64, num); +} + void httpJsonTimestamp(JsonBuf* buf, int64_t t, bool us) { char ts[35] = {0}; struct tm* ptm; @@ -303,6 +309,12 @@ void httpJsonInt(JsonBuf* buf, int32_t num) { buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%d", num); } +void httpJsonUInt(JsonBuf* buf, uint32_t num) { + httpJsonItemToken(buf); + httpJsonTestBuf(buf, MAX_NUM_STR_SZ); + buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%u", num); +} + void httpJsonFloat(JsonBuf* buf, float num) { httpJsonItemToken(buf); httpJsonTestBuf(buf, MAX_NUM_STR_SZ); diff --git a/src/plugins/http/src/httpRestJson.c b/src/plugins/http/src/httpRestJson.c index 61a5a361c4865d6b3dbac73773cc0b8cfc562e56..60c23e603e5107e858db2138d1aed87d5f8a5e1b 100644 --- a/src/plugins/http/src/httpRestJson.c +++ b/src/plugins/http/src/httpRestJson.c @@ -162,6 +162,18 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, case TSDB_DATA_TYPE_BIGINT: httpJsonInt64(jsonBuf, *((int64_t *)row[i])); break; + case TSDB_DATA_TYPE_UTINYINT: + httpJsonUInt(jsonBuf, *((uint8_t *)row[i])); + break; + case TSDB_DATA_TYPE_USMALLINT: + httpJsonUInt(jsonBuf, *((uint16_t *)row[i])); + break; + case TSDB_DATA_TYPE_UINT: + httpJsonUInt(jsonBuf, *((uint32_t *)row[i])); + break; + case TSDB_DATA_TYPE_UBIGINT: + httpJsonUInt64(jsonBuf, *((uint64_t *)row[i])); + break; case TSDB_DATA_TYPE_FLOAT: httpJsonFloat(jsonBuf, GET_FLOAT_VAL(row[i])); break; diff --git a/src/query/inc/qAggMain.h b/src/query/inc/qAggMain.h index 7122f63593c64a6bffd6f3d53cd8ef21eca800ff..c59067c4b357aa084cd6f31c39f3038075d18a82 100644 --- a/src/query/inc/qAggMain.h +++ b/src/query/inc/qAggMain.h @@ -26,6 +26,7 @@ extern "C" { #include "taosdef.h" #include "trpc.h" #include "tvariant.h" +#include "tsdb.h" #define TSDB_FUNC_INVALID_ID -1 #define TSDB_FUNC_COUNT 0 @@ -70,15 +71,17 @@ extern "C" { #define TSDB_FUNC_AVG_IRATE 34 #define TSDB_FUNC_TID_TAG 35 -#define TSDB_FUNC_HISTOGRAM 36 -#define TSDB_FUNC_HLL 37 -#define TSDB_FUNC_MODE 38 -#define TSDB_FUNC_SAMPLE 39 -#define TSDB_FUNC_CEIL 40 -#define TSDB_FUNC_FLOOR 41 -#define TSDB_FUNC_ROUND 42 -#define TSDB_FUNC_MAVG 43 -#define TSDB_FUNC_CSUM 44 +#define TSDB_FUNC_BLKINFO 36 + +#define TSDB_FUNC_HISTOGRAM 37 +#define TSDB_FUNC_HLL 38 +#define TSDB_FUNC_MODE 39 +#define TSDB_FUNC_SAMPLE 40 +#define TSDB_FUNC_CEIL 41 +#define TSDB_FUNC_FLOOR 42 +#define TSDB_FUNC_ROUND 43 +#define TSDB_FUNC_MAVG 44 +#define TSDB_FUNC_CSUM 45 #define TSDB_FUNCSTATE_SO 0x1u // single output @@ -214,13 +217,14 @@ typedef struct SAggFunctionInfo { void (*xFinalize)(SQLFunctionCtx *pCtx); void (*mergeFunc)(SQLFunctionCtx *pCtx); - int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId); + int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId); } SAggFunctionInfo; #define GET_RES_INFO(ctx) ((ctx)->resultInfo) int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type, int16_t *len, int32_t *interBytes, int16_t extLength, bool isSuperTable); +int32_t isValidFunction(const char* name, int32_t len); #define IS_STREAM_QUERY_VALID(x) (((x)&TSDB_FUNCSTATE_STREAM) != 0) #define IS_MULTIOUTPUT(x) (((x)&TSDB_FUNCSTATE_MO) != 0) @@ -242,12 +246,16 @@ typedef struct STwaInfo { STimeWindow win; } STwaInfo; +struct SBufferWriter; +void blockDistInfoToBinary(STableBlockDist* pDist, struct SBufferWriter* bw); +void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDist); + /* global sql function array */ extern struct SAggFunctionInfo aAggs[]; extern int32_t functionCompatList[]; // compatible check array list -bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const char *minval, const char *maxval); +bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const char *maxval); /** * the numOfRes should be kept, since it may be used later @@ -258,14 +266,14 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const cha (_r)->initialized = false; \ } while (0) -static FORCE_INLINE void initResultInfo(SResultRowCellInfo *pResInfo, uint32_t bufLen) { +static FORCE_INLINE void initResultInfo(SResultRowCellInfo *pResInfo, int32_t bufLen) { pResInfo->initialized = true; // the this struct has been initialized flag pResInfo->complete = false; pResInfo->hasResult = false; pResInfo->numOfRes = 0; - memset(GET_ROWCELL_INTERBUF(pResInfo), 0, (size_t)bufLen); + memset(GET_ROWCELL_INTERBUF(pResInfo), 0, bufLen); } #ifdef __cplusplus diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 0d296b365e40372ee1d48407b148ec96278d42fc..5ff574ec673119d3837ef57d119c778e6cd8c285 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -12,8 +12,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef TDENGINE_QUERYEXECUTOR_H -#define TDENGINE_QUERYEXECUTOR_H +#ifndef TDENGINE_QEXECUTOR_H +#define TDENGINE_QEXECUTOR_H #include "os.h" @@ -37,30 +37,24 @@ typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int #define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0u) #define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP) -#define SET_STABLE_QUERY_OVER(_q) ((_q)->tableIndex = (int32_t)((_q)->tableqinfoGroupInfo.numOfTables)) -#define IS_STASBLE_QUERY_OVER(_q) ((_q)->tableIndex >= (int32_t)((_q)->tableqinfoGroupInfo.numOfTables)) - #define GET_TABLEGROUP(q, _index) ((SArray*) taosArrayGetP((q)->tableqinfoGroupInfo.pGroupList, (_index))) +#define GET_NUM_OF_RESULTS(_r) (((_r)->outputBuf) == NULL? 0:((_r)->outputBuf)->info.rows) + enum { // when query starts to execute, this status will set QUERY_NOT_COMPLETED = 0x1u, - /* result output buffer is full, current query is paused. - * this status is only exist in group-by clause and diff/add/division/multiply/ query. - */ - QUERY_RESBUF_FULL = 0x2u, - /* query is over * 1. this status is used in one row result query process, e.g., count/sum/first/last/ avg...etc. * 2. when all data within queried time window, it is also denoted as query_completed */ - QUERY_COMPLETED = 0x4u, + QUERY_COMPLETED = 0x2u, /* when the result is not completed return to client, this status will be * usually used in case of interval query with interpolation option */ - QUERY_OVER = 0x8u, + QUERY_OVER = 0x4u, }; typedef struct SResultRowPool { @@ -86,13 +80,13 @@ typedef struct SSqlGroupbyExpr { typedef struct SResultRow { int32_t pageId; // pageId & rowId is the position of current result in disk-based output buffer - int32_t rowId:29; // row index in buffer page + int32_t offset:29; // row index in buffer page bool startInterp; // the time window start timestamp has done the interpolation already. bool endInterp; // the time window end timestamp has done the interpolation already. bool closed; // this result status: closed or opened uint32_t numOfRows; // number of rows of current time window SResultRowCellInfo* pCellInfo; // For each result column, there is a resultInfo - union {STimeWindow win; char* key;}; // start key of current time window + union {STimeWindow win; char* key;}; // start key of current result row } SResultRow; typedef struct SGroupResInfo { @@ -106,12 +100,11 @@ typedef struct SGroupResInfo { * If the number of generated results is greater than this value, * query query will be halt and return results to client immediate. */ -typedef struct SResultRec { +typedef struct SRspResultInfo { int64_t total; // total generated result size in rows - int64_t rows; // current result set size in rows - int64_t capacity; // capacity of current result output buffer + int32_t capacity; // capacity of current result output buffer int32_t threshold; // result size threshold in rows. -} SResultRec; +} SRspResultInfo; typedef struct SResultRowInfo { SResultRow** pResult; // result list @@ -138,7 +131,6 @@ typedef struct SSingleColumnFilterInfo { typedef struct STableQueryInfo { TSKEY lastKey; int32_t groupIndex; // group id in table list - int16_t queryRangeSet; // denote if the query range is set, only available for interval query tVariant tag; STimeWindow win; STSCursor cur; @@ -179,82 +171,136 @@ typedef struct { SArray* pResult; // SArray } SInterResult; +typedef struct SSDataBlock { + SDataStatis *pBlockStatis; + SArray *pDataBlock; + SDataBlockInfo info; +} SSDataBlock; + typedef struct SQuery { + SLimitVal limit; + + bool stableQuery; // super table query or not + bool topBotQuery; // TODO used bitwise flag + bool groupbyColumn; // denote if this is a groupby normal column query + bool hasTagResults; // if there are tag values in final result or not + bool timeWindowInterpo;// if the time window start/end required interpolation + bool queryBlockDist; // if query data block distribution + bool stabledev; // super table stddev query + int32_t interBufSize; // intermediate buffer sizse + + SOrderVal order; int16_t numOfCols; int16_t numOfTags; - SOrderVal order; + STimeWindow window; SInterval interval; + SSessionWindow sw; int16_t precision; int16_t numOfOutput; int16_t fillType; int16_t checkResultBuf; // check if the buffer is full during scan each block - SLimitVal limit; int32_t srcRowSize; // todo extract struct int32_t resultRowSize; + int32_t intermediateResultRowSize; // intermediate result row size, in case of top-k query. int32_t maxSrcColumnSize; int32_t tagLen; // tag value length of current query - SSqlGroupbyExpr* pGroupbyExpr; SExprInfo* pExpr1; SExprInfo* pExpr2; int32_t numOfExpr2; - SColumnInfo* colList; SColumnInfo* tagColList; int32_t numOfFilterCols; int64_t* fillVal; - uint32_t status; // query status - SResultRec rec; - int32_t pos; - tFilePage** sdata; - STableQueryInfo* current; - int32_t numOfCheckedBlocks; // number of check data blocks - SOrderedPrjQueryInfo prjInfo; // limit value for each vgroup, only available in global order projection query. SSingleColumnFilterInfo* pFilterInfo; + + STableQueryInfo* current; + void* tsdb; + SMemRef memRef; + STableGroupInfo tableGroupInfo; // table list SArray + int32_t vgId; } SQuery; +typedef SSDataBlock* (*__operator_fn_t)(void* param); +typedef void (*__optr_cleanup_fn_t)(void* param, int32_t num); + +struct SOperatorInfo; + typedef struct SQueryRuntimeEnv { - jmp_buf env; - SQuery* pQuery; - SQLFunctionCtx* pCtx; - int32_t numOfRowsPerPage; - uint16_t* offset; - uint16_t scanFlag; // denotes reversed scan of data or not - SFillInfo* pFillInfo; - SResultRowInfo resultRowInfo; - - SQueryCostInfo summary; - void* pQueryHandle; - void* pSecQueryHandle; // another thread for - bool stableQuery; // super table query or not - bool topBotQuery; // TODO used bitwise flag - bool groupbyColumn; // denote if this is a groupby normal column query - bool hasTagResults; // if there are tag values in final result or not - bool timeWindowInterpo;// if the time window start/end required interpolation - bool queryWindowIdentical; // all query time windows are identical for all tables in one group - bool queryBlockDist; // if query data block distribution - bool stabledev; // super table stddev query - int32_t interBufSize; // intermediate buffer sizse - int32_t prevGroupId; // previous executed group id - SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file - SHashObj* pResultRowHashTable; // quick locate the window object for each result - char* keyBuf; // window key buffer - SResultRowPool* pool; // window result object pool - - int32_t* rowCellInfoOffset;// offset value for each row result cell info - char** prevRow; - - SArray* prevResult; // intermediate result, SArray - STSBuf* pTsBuf; // timestamp filter list - STSCursor cur; - - char* tagVal; // tag value of current data block - SArithmeticSupport *sasArray; + jmp_buf env; + SQuery* pQuery; + uint32_t status; // query status + void* qinfo; + uint8_t scanFlag; // denotes reversed scan of data or not + void* pQueryHandle; + + int32_t prevGroupId; // previous executed group id + SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file + SHashObj* pResultRowHashTable; // quick locate the window object for each result + char* keyBuf; // window key buffer + SResultRowPool* pool; // window result object pool + char** prevRow; + + SArray* prevResult; // intermediate result, SArray + STSBuf* pTsBuf; // timestamp filter list + STSCursor cur; + + char* tagVal; // tag value of current data block + SArithmeticSupport *sasArray; + + SSDataBlock *outputBuf; + STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure + struct SOperatorInfo *proot; + struct SOperatorInfo *pTableScanner; // table scan operator + SGroupResInfo groupResInfo; + int64_t currentOffset; // dynamic offset value + + SRspResultInfo resultInfo; + SHashObj *pTableRetrieveTsMap; } SQueryRuntimeEnv; +enum { + OP_IN_EXECUTING = 1, + OP_RES_TO_RETURN = 2, + OP_EXEC_DONE = 3, +}; + +enum OPERATOR_TYPE_E { + OP_TableScan = 1, + OP_DataBlocksOptScan = 2, + OP_TableSeqScan = 3, + OP_TagScan = 4, + OP_TableBlockInfoScan= 5, + OP_Aggregate = 6, + OP_Arithmetic = 7, + OP_Groupby = 8, + OP_Limit = 9, + OP_Offset = 10, + OP_TimeWindow = 11, + OP_SessionWindow = 12, + OP_Fill = 13, + OP_MultiTableAggregate = 14, + OP_MultiTableTimeInterval = 15, +}; + +typedef struct SOperatorInfo { + uint8_t operatorType; + bool blockingOptr; // block operator or not + uint8_t status; // denote if current operator is completed + int32_t numOfOutput; // number of columns of the current operator results + char *name; // name, used to show the query execution plan + void *info; // extension attribution + SExprInfo *pExpr; + SQueryRuntimeEnv *pRuntimeEnv; + + struct SOperatorInfo *upstream; + __operator_fn_t exec; + __optr_cleanup_fn_t cleanup; +} SOperatorInfo; + enum { QUERY_RESULT_NOT_READY = 1, QUERY_RESULT_READY = 2, @@ -263,23 +309,11 @@ enum { typedef struct SQInfo { void* signature; uint64_t qId; - int32_t code; // error code to returned to client - int64_t owner; // if it is in execution - void* tsdb; - SMemRef memRef; - int32_t vgId; - STableGroupInfo tableGroupInfo; // table list SArray - STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure - SQueryRuntimeEnv runtimeEnv; - SHashObj* arrTableIdInfo; - int32_t groupIndex; + int32_t code; // error code to returned to client + int64_t owner; // if it is in execution - /* - * the query is executed position on which meter of the whole list. - * when the index reaches the last one of the list, it means the query is completed. - */ - int32_t tableIndex; - SGroupResInfo groupResInfo; + SQueryRuntimeEnv runtimeEnv; + SQuery query; void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables; pthread_mutex_t lock; // used to synchronize the rsp/query threads @@ -288,6 +322,7 @@ typedef struct SQInfo { void* rspContext; // response context int64_t startExecTs; // start to exec timestamp char* sql; // query sql string + SQueryCostInfo summary; } SQInfo; typedef struct SQueryParam { @@ -306,10 +341,93 @@ typedef struct SQueryParam { SSqlGroupbyExpr *pGroupbyExpr; } SQueryParam; +typedef struct STableScanInfo { + void *pQueryHandle; + int32_t numOfBlocks; + int32_t numOfSkipped; + int32_t numOfBlockStatis; + int64_t numOfRows; + + int32_t order; // scan order + int32_t times; // repeat counts + int32_t current; + int32_t reverseTimes; // 0 by default + + SQLFunctionCtx *pCtx; // next operator query context + SResultRowInfo *pResultRowInfo; + int32_t *rowCellInfoOffset; + SExprInfo *pExpr; + SSDataBlock block; + bool loadExternalRows; // load external rows (prev & next rows) + int32_t numOfOutput; + int64_t elapsedTime; + + int32_t tableIndex; +} STableScanInfo; + +typedef struct STagScanInfo { + SColumnInfo* pCols; + SSDataBlock* pRes; + int32_t totalTables; + int32_t currentIndex; +} STagScanInfo; + +typedef struct SOptrBasicInfo { + SResultRowInfo resultRowInfo; + int32_t *rowCellInfoOffset; // offset value for each row result cell info + SQLFunctionCtx *pCtx; + SSDataBlock *pRes; +} SOptrBasicInfo; + +typedef struct SOptrBasicInfo STableIntervalOperatorInfo; + +typedef struct SAggOperatorInfo { + SOptrBasicInfo binfo; + uint32_t seed; +} SAggOperatorInfo; + +typedef struct SArithOperatorInfo { + SOptrBasicInfo binfo; + int32_t bufCapacity; + uint32_t seed; +} SArithOperatorInfo; + +typedef struct SLimitOperatorInfo { + int64_t limit; + int64_t total; +} SLimitOperatorInfo; + +typedef struct SOffsetOperatorInfo { + int64_t offset; +} SOffsetOperatorInfo; + +typedef struct SFillOperatorInfo { + SFillInfo *pFillInfo; + SSDataBlock *pRes; + int64_t totalInputRows; +} SFillOperatorInfo; + +typedef struct SGroupbyOperatorInfo { + SOptrBasicInfo binfo; + int32_t colIndex; + char *prevData; // previous group by value +} SGroupbyOperatorInfo; + +typedef struct SSWindowOperatorInfo { + SOptrBasicInfo binfo; + STimeWindow curWindow; // current time window + TSKEY prevTs; // previous timestamp + int32_t numOfRows; // number of rows + int32_t start; // start row index +} SSWindowOperatorInfo; + void freeParam(SQueryParam *param); int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param); int32_t createQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo, SSqlFuncMsg **pExprMsg, SColumnInfo* pTagCols); +int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo, + SSqlFuncMsg **pExprMsg, SExprInfo *prevExpr); + SSqlGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code); SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, bool stableQuery, char* sql, uint64_t *qId); @@ -319,13 +437,9 @@ void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters); bool isQueryKilled(SQInfo *pQInfo); int32_t checkForQueryBuf(size_t numOfTables); bool doBuildResCheck(SQInfo* pQInfo); -void setQueryStatus(SQuery *pQuery, int8_t status); +void setQueryStatus(SQueryRuntimeEnv *pRuntimeEnv, int8_t status); bool onlyQueryTags(SQuery* pQuery); -void buildTagQueryResult(SQInfo *pQInfo); -void stableQueryImpl(SQInfo *pQInfo); -void buildTableBlockDistResult(SQInfo *pQInfo); -void tableQueryImpl(SQInfo *pQInfo); bool isValidQInfo(void *param); int32_t doDumpQueryResult(SQInfo *pQInfo, char *data); @@ -337,4 +451,4 @@ void freeQInfo(SQInfo *pQInfo); int32_t getMaximumIdleDurationSec(); -#endif // TDENGINE_QUERYEXECUTOR_H +#endif // TDENGINE_QEXECUTOR_H diff --git a/src/query/inc/qFill.h b/src/query/inc/qFill.h index aa6df9279acb9e3cecf20d71f726974ee89a4030..00ac86caf4a079e01ad239496370d97ac28e84f2 100644 --- a/src/query/inc/qFill.h +++ b/src/query/inc/qFill.h @@ -24,6 +24,8 @@ extern "C" { #include "qExtbuffer.h" #include "taosdef.h" +struct SSDataBlock; + typedef struct { STColumn col; // column info int16_t functionId; // sql function id @@ -78,7 +80,7 @@ void* taosDestroyFillInfo(SFillInfo *pFillInfo); void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); -void taosFillSetDataBlockFromFilePage(SFillInfo* pFillInfo, const tFilePage** pInput); +void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const struct SSDataBlock* pInput); void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, const tFilePage* pInput); @@ -88,7 +90,7 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType); -int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity); +int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t capacity); #ifdef __cplusplus } diff --git a/src/query/inc/qResultbuf.h b/src/query/inc/qResultbuf.h index 704df9f3f29cc0653c353ed194e02e72cb4c5fb2..e46af8c29891cf52fbbac97c6afb42e0d0571215 100644 --- a/src/query/inc/qResultbuf.h +++ b/src/query/inc/qResultbuf.h @@ -55,7 +55,6 @@ typedef struct SResultBufStatis { } SResultBufStatis; typedef struct SDiskbasedResultBuf { - int32_t numOfRowsPerPage; int32_t numOfPages; int64_t totalBufSize; int64_t fileSize; // disk file size @@ -77,7 +76,7 @@ typedef struct SDiskbasedResultBuf { SResultBufStatis statis; } SDiskbasedResultBuf; -#define DEFAULT_INTERN_BUF_PAGE_SIZE (256L) // in bytes +#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L) // in bytes #define PAGE_INFO_INITIALIZER (SPageDiskInfo){-1, -1} /** @@ -89,8 +88,7 @@ typedef struct SDiskbasedResultBuf { * @param handle * @return */ -int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t rowSize, int32_t pagesize, - int32_t inMemBufSize, const void* handle); +int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t pagesize, int32_t inMemBufSize, const void* handle); /** * @@ -101,13 +99,6 @@ int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t ro */ tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId); -/** - * - * @param pResultBuf - * @return - */ -size_t getNumOfRowsPerPage(const SDiskbasedResultBuf* pResultBuf); - /** * * @param pResultBuf diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h index 30c6a361776547267c6274751d8bda7af09706b0..53c9a6dc2c49427dcf8388b1b458a37ce11e26f9 100644 --- a/src/query/inc/qSqlparser.h +++ b/src/query/inc/qSqlparser.h @@ -45,7 +45,7 @@ typedef struct SLimitVal { typedef struct SOrderVal { uint32_t order; - int32_t orderColId; + int32_t orderColId; } SOrderVal; typedef struct tVariantListItem { @@ -58,14 +58,19 @@ typedef struct SIntervalVal { SStrToken offset; } SIntervalVal; +typedef struct SSessionWindowVal { + SStrToken col; + SStrToken gap; +} SSessionWindowVal; + typedef struct SQuerySQL { struct tSQLExprList *pSelection; // select clause SArray * from; // from clause SArray struct tSQLExpr * pWhere; // where clause [optional] SArray * pGroupby; // groupby clause, only for tags[optional], SArray SArray * pSortOrder; // orderby [optional], SArray - SStrToken interval; // interval [optional] - SStrToken offset; // offset window [optional] + SIntervalVal interval; // (interval, interval_offset) [optional] + SSessionWindowVal sessionVal; // session window [optional] SStrToken sliding; // sliding window [optional] SLimitVal limit; // limit offset [optional] SLimitVal slimit; // group limit offset [optional] @@ -193,19 +198,32 @@ typedef struct SSqlInfo { }; } SSqlInfo; +#define NON_ARITHMEIC_EXPR 0 +#define NORMAL_ARITHMETIC 1 +#define AGG_ARIGHTMEIC 2 + +enum SQL_NODE_TYPE { + SQL_NODE_TABLE_COLUMN= 1, + SQL_NODE_SQLFUNCTION = 2, + SQL_NODE_VALUE = 3, + SQL_NODE_EXPR = 4, +}; + typedef struct tSQLExpr { - uint32_t nSQLOptr; // TK_FUNCTION: sql function, TK_LE: less than(binary expr) - - // the full sql string of function(col, param), which is actually the raw - // field name, since the function name is kept in nSQLOptr already + uint16_t type; // sql node type + uint32_t tokenId; // TK_FUNCTION: sql function, TK_LE: less than(binary expr) + + // the whole string of the function(col, param), while the function name is kept in token SStrToken operand; - SStrToken colInfo; // field id - tVariant val; // value only for string, float, int + uint32_t functionId; // function id + + SStrToken colInfo; // table column info + tVariant value; // the use input value SStrToken token; // original sql expr string struct tSQLExpr *pLeft; // left child struct tSQLExpr *pRight; // right child - struct tSQLExprList *pParam; // function parameters + struct tSQLExprList *pParam; // function parameters list } tSQLExpr; // used in select clause. select from xxx @@ -251,8 +269,8 @@ tSQLExprList *tSqlExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken void tSqlExprListDestroy(tSQLExprList *pList); -SQuerySQL *tSetQuerySqlElems(SStrToken *pSelectToken, tSQLExprList *pSelection, SArray *pFrom, tSQLExpr *pWhere, - SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, +SQuerySQL *tSetQuerySqlNode(SStrToken *pSelectToken, tSQLExprList *pSelection, SArray *pFrom, tSQLExpr *pWhere, + SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, SSessionWindowVal *pSession, SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, SLimitVal *pGLimit); SCreateTableSQL *tSetCreateSqlElems(SArray *pCols, SArray *pTags, SQuerySQL *pSelect, int32_t type); @@ -302,16 +320,6 @@ void tSqlSetColumnType(TAOS_FIELD *pField, SStrToken *type); void *ParseAlloc(void *(*mallocProc)(size_t)); -enum { - TSQL_NODE_TYPE_EXPR = 0x1, - TSQL_NODE_TYPE_ID = 0x2, - TSQL_NODE_TYPE_VALUE = 0x4, -}; - -#define NON_ARITHMEIC_EXPR 0 -#define NORMAL_ARITHMETIC 1 -#define AGG_ARIGHTMEIC 2 - SSqlInfo qSQLParse(const char *str); #ifdef __cplusplus diff --git a/src/query/inc/qTsbuf.h b/src/query/inc/qTsbuf.h index 5d055782c9b82a1444c97a62d429cc2ba9a53986..00cc4e897f130348b81a7d96419c1b292cacca8c 100644 --- a/src/query/inc/qTsbuf.h +++ b/src/query/inc/qTsbuf.h @@ -112,13 +112,11 @@ STSBuf* tsBufClone(STSBuf* pTSBuf); STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id); -void tsBufFlush(STSBuf* pTSBuf); - +void tsBufFlush(STSBuf* pTSBuf); void tsBufResetPos(STSBuf* pTSBuf); -STSElem tsBufGetElem(STSBuf* pTSBuf); - bool tsBufNextPos(STSBuf* pTSBuf); +STSElem tsBufGetElem(STSBuf* pTSBuf); STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t id, tVariant* tag); STSCursor tsBufGetCursor(STSBuf* pTSBuf); diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index d4a0c25886ad63ff5cc3e79cb0a9e84c156197cd..31dfc350a39b642631c1e9c60f713e953d6898c3 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -27,7 +27,7 @@ #define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t)) #define curTimeWindowIndex(_winres) ((_winres)->curIndex) -#define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!sq))? (_q)->pExpr1[1].base.arg->argValue.i64:1) +#define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!(sq)))? (_q)->pExpr1[1].base.arg->argValue.i64:1) int32_t getOutputInterResultBufSize(SQuery* pQuery); @@ -44,22 +44,18 @@ void closeResultRow(SResultRowInfo* pResultRowInfo, int32_t slot); bool isResultRowClosed(SResultRowInfo *pResultRowInfo, int32_t slot); void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pResultRow, int16_t type); -SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index); +SResultRowCellInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset); static FORCE_INLINE SResultRow *getResultRow(SResultRowInfo *pResultRowInfo, int32_t slot) { assert(pResultRowInfo != NULL && slot >= 0 && slot < pResultRowInfo->size); return pResultRowInfo->pResult[slot]; } -static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SResultRow *pResult, - tFilePage* page) { - assert(pResult != NULL && pRuntimeEnv != NULL); +static FORCE_INLINE char *getPosInResultPage(SQuery *pQuery, tFilePage* page, int32_t rowOffset, int16_t offset) { + assert(rowOffset >= 0 && pQuery != NULL); - SQuery *pQuery = pRuntimeEnv->pQuery; - - int32_t realRowId = (int32_t)(pResult->rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery)); - return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * pRuntimeEnv->numOfRowsPerPage + - pQuery->pExpr1[columnIndex].bytes * realRowId; + int32_t numOfRows = (int32_t)GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pQuery->topBotQuery, pQuery->stableQuery); + return ((char *)page->data) + rowOffset + offset * numOfRows; } bool isNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type); @@ -74,8 +70,6 @@ void* destroyResultRowPool(SResultRowPool* p); int32_t getNumOfAllocatedResultRows(SResultRowPool* p); int32_t getNumOfUsedResultRows(SResultRowPool* p); -bool isPointInterpoQuery(SQuery *pQuery); - typedef struct { SArray* pResult; // SArray int32_t colId; @@ -85,12 +79,14 @@ void interResToBinary(SBufferWriter* bw, SArray* pRes, int32_t tagLen); SArray* interResFromBinary(const char* data, int32_t len); void freeInterResult(void* param); -void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo, int32_t offset); +void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo); void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo); +bool hasRemainDataInCurrentGroup(SGroupResInfo* pGroupResInfo); bool hasRemainData(SGroupResInfo* pGroupResInfo); + bool incNextGroup(SGroupResInfo* pGroupResInfo); int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); -int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, SQInfo *pQInfo); +int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, SQueryRuntimeEnv *pRuntimeEnv, int32_t* offset); #endif // TDENGINE_QUERYUTIL_H diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index cb0c94708e43ab38ef6eed1e5108aaa1164435bb..85e6017dc46882345b92d43c5a6c3b5d386d4886 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -329,8 +329,8 @@ signed(A) ::= PLUS INTEGER(X). { A = strtol(X.z, NULL, 10); } signed(A) ::= MINUS INTEGER(X). { A = -strtol(X.z, NULL, 10);} ////////////////////////////////// The CREATE TABLE statement /////////////////////////////// -cmd ::= CREATE TABLE create_table_args. {} -cmd ::= CREATE TABLE create_stable_args. {} +cmd ::= CREATE TABLE create_table_args. {} +cmd ::= CREATE TABLE create_stable_args. {} cmd ::= CREATE STABLE create_stable_args. {} cmd ::= CREATE TABLE create_table_list(Z). { pInfo->type = TSDB_SQL_CREATE_TABLE; pInfo->pCreateTableInfo = Z;} @@ -455,8 +455,8 @@ tagitem(A) ::= PLUS(X) FLOAT(Y). { //////////////////////// The SELECT statement ///////////////////////////////// %type select {SQuerySQL*} %destructor select {doDestroyQuerySql($$);} -select(A) ::= SELECT(T) selcollist(W) from(X) where_opt(Y) interval_opt(K) fill_opt(F) sliding_opt(S) groupby_opt(P) orderby_opt(Z) having_opt(N) slimit_opt(G) limit_opt(L). { - A = tSetQuerySqlElems(&T, W, X, Y, P, Z, &K, &S, F, &L, &G); +select(A) ::= SELECT(T) selcollist(W) from(X) where_opt(Y) interval_opt(K) session_option(H) fill_opt(F) sliding_opt(S) groupby_opt(P) orderby_opt(Z) having_opt(N) slimit_opt(G) limit_opt(L). { + A = tSetQuerySqlNode(&T, W, X, Y, P, Z, &K, &H, &S, F, &L, &G); } %type union {SSubclauseInfo*} @@ -474,7 +474,7 @@ cmd ::= union(X). { setSqlInfo(pInfo, X, NULL, TSDB_SQL_SELECT); } // select server_version(), select client_version(), // select server_state(); select(A) ::= SELECT(T) selcollist(W). { - A = tSetQuerySqlElems(&T, W, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); + A = tSetQuerySqlNode(&T, W, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); } // selcollist is a list of expressions that are to become the return @@ -549,13 +549,21 @@ tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z) ids(F). { tmvar(A) ::= VARIABLE(X). {A = X;} %type interval_opt {SIntervalVal} -interval_opt(N) ::= INTERVAL LP tmvar(E) RP. {N.interval = E; N.offset.n = 0; N.offset.z = NULL; N.offset.type = 0;} -interval_opt(N) ::= INTERVAL LP tmvar(E) COMMA tmvar(O) RP. {N.interval = E; N.offset = O;} +interval_opt(N) ::= INTERVAL LP tmvar(E) RP. {N.interval = E; N.offset.n = 0;} +interval_opt(N) ::= INTERVAL LP tmvar(E) COMMA tmvar(X) RP. {N.interval = E; N.offset = X;} interval_opt(N) ::= . {memset(&N, 0, sizeof(N));} +%type session_option {SSessionWindowVal} +session_option(X) ::= . {X.col.n = 0; X.gap.n = 0;} +session_option(X) ::= SESSION LP ids(V) cpxName(Z) COMMA tmvar(Y) RP. { + V.n += Z.n; + X.col = V; + X.gap = Y; +} + %type fill_opt {SArray*} %destructor fill_opt {taosArrayDestroy($$);} -fill_opt(N) ::= . {N = 0; } +fill_opt(N) ::= . { N = 0; } fill_opt(N) ::= FILL LP ID(Y) COMMA tagitemlist(X) RP. { tVariant A = {0}; toTSDBType(Y.type); @@ -837,6 +845,5 @@ cmd ::= KILL QUERY INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); s %fallback ID ABORT AFTER ASC ATTACH BEFORE BEGIN CASCADE CLUSTER CONFLICT COPY DATABASE DEFERRED DELIMITERS DESC DETACH EACH END EXPLAIN FAIL FOR GLOB IGNORE IMMEDIATE INITIALLY INSTEAD LIKE MATCH KEY OF OFFSET RAISE REPLACE RESTRICT ROW STATEMENT TRIGGER VIEW ALL - COUNT SUM AVG MIN MAX FIRST LAST TOP BOTTOM STDDEV PERCENTILE APERCENTILE LEASTSQUARES HISTOGRAM DIFF - SPREAD TWA INTERP LAST_ROW RATE IRATE SUM_RATE SUM_IRATE AVG_RATE AVG_IRATE TBID NOW IPTOKEN SEMI NONE PREV LINEAR IMPORT + NOW IPTOKEN SEMI NONE PREV LINEAR IMPORT METRIC TBNAME JOIN METRICS STABLE NULL INSERT INTO VALUES. diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index f99fe3f0720784db0a5e0b526dddb436a91f87d3..f18d093b89d48419e9707d606a289b3168fea8bf 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -18,6 +18,7 @@ #include "taosmsg.h" #include "texpr.h" #include "ttype.h" +#include "tsdb.h" #include "qAggMain.h" #include "qFill.h" @@ -26,11 +27,9 @@ #include "qTsbuf.h" #include "queryLog.h" -//#define GET_INPUT_DATA_LIST(x) (((char *)((x)->pInput)) + ((x)->startOffset) * ((x)->inputBytes)) #define GET_INPUT_DATA_LIST(x) ((char *)((x)->pInput)) #define GET_INPUT_DATA(x, y) (GET_INPUT_DATA_LIST(x) + (y) * (x)->inputBytes) -//#define GET_TS_LIST(x) ((TSKEY*)&((x)->ptsList[(x)->startOffset])) #define GET_TS_LIST(x) ((TSKEY*)((x)->ptsList)) #define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)]) @@ -191,6 +190,11 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI *bytes = (int16_t)(dataBytes + sizeof(int16_t) + sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t) + VARSTR_HEADER_SIZE); *interBytes = 0; return TSDB_CODE_SUCCESS; + } else if (functionId == TSDB_FUNC_BLKINFO) { + *type = TSDB_DATA_TYPE_BINARY; + *bytes = 16384; + *interBytes = 0; + return TSDB_CODE_SUCCESS; } if (functionId == TSDB_FUNC_COUNT) { @@ -209,7 +213,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI if (functionId == TSDB_FUNC_TS_COMP) { *type = TSDB_DATA_TYPE_BINARY; - *bytes = sizeof(int32_t); // this results is compressed ts data + *bytes = 1; // this results is compressed ts data, only one byte *interBytes = POINTER_BYTES; return TSDB_CODE_SUCCESS; } @@ -355,6 +359,22 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI return TSDB_CODE_SUCCESS; } +// TODO use hash table +int32_t isValidFunction(const char* name, int32_t len) { + for(int32_t i = 0; i <= TSDB_FUNC_BLKINFO; ++i) { + int32_t nameLen = (int32_t) strlen(aAggs[i].name); + if (len != nameLen) { + continue; + } + + if (strncasecmp(aAggs[i].name, name, len) == 0) { + return i; + } + } + + return -1; +} + // set the query flag to denote that query is completed static void no_next_step(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); @@ -458,7 +478,7 @@ static void count_func_merge(SQLFunctionCtx *pCtx) { * @param filterCols * @return */ -int32_t count_load_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { +int32_t countRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { if (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { return BLK_DATA_NO_NEEDED; } else { @@ -466,7 +486,7 @@ int32_t count_load_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32 } } -int32_t no_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { +int32_t noDataRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { return BLK_DATA_NO_NEEDED; } @@ -674,16 +694,16 @@ static void sum_func_merge(SQLFunctionCtx *pCtx) { } } -static int32_t statisRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { +static int32_t statisRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { return BLK_DATA_STATIS_NEEDED; } -static int32_t dataBlockRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { +static int32_t dataBlockRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { return BLK_DATA_ALL_NEEDED; } -// todo: if column in current data block are null, opt for this case -static int32_t firstFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { +// todo: if column in current data block are null, opt for this case +static int32_t firstFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { if (pCtx->order == TSDB_ORDER_DESC) { return BLK_DATA_NO_NEEDED; } @@ -696,7 +716,7 @@ static int32_t firstFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, i } } -static int32_t lastFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { +static int32_t lastFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { if (pCtx->order != pCtx->param[0].i64) { return BLK_DATA_NO_NEEDED; } @@ -708,7 +728,7 @@ static int32_t lastFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, in } } -static int32_t firstDistFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { +static int32_t firstDistFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { if (pCtx->order == TSDB_ORDER_DESC) { return BLK_DATA_NO_NEEDED; } @@ -724,11 +744,11 @@ static int32_t firstDistFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY en if (pInfo->hasResult != DATA_SET_FLAG) { return BLK_DATA_ALL_NEEDED; } else { // data in current block is not earlier than current result - return (pInfo->ts <= start) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; + return (pInfo->ts <= w->skey) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; } } -static int32_t lastDistFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { +static int32_t lastDistFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { if (pCtx->order != pCtx->param[0].i64) { return BLK_DATA_NO_NEEDED; } @@ -744,7 +764,7 @@ static int32_t lastDistFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end if (pInfo->hasResult != DATA_SET_FLAG) { return BLK_DATA_ALL_NEEDED; } else { - return (pInfo->ts > end) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; + return (pInfo->ts > w->ekey) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; } } @@ -1359,7 +1379,21 @@ static void min_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void stddev_function(SQLFunctionCtx *pCtx) { - SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo); + + if (pCtx->currentStage == REPEAT_SCAN && pStd->stage == 0) { + pStd->stage++; + avg_finalizer(pCtx); + + pResInfo->initialized = true; // set it initialized to avoid re-initialization + + // save average value into tmpBuf, for second stage scan + SAvgInfo *pAvg = GET_ROWCELL_INTERBUF(pResInfo); + + pStd->avg = GET_DOUBLE_VAL(pCtx->pOutput); + assert((isnan(pAvg->sum) && pAvg->num == 0) || (pStd->num == pAvg->num && pStd->avg == pAvg->sum)); + } if (pStd->stage == 0) { // the first stage is to calculate average value @@ -1432,7 +1466,20 @@ static void stddev_function_f(SQLFunctionCtx *pCtx, int32_t index) { // the second stage to calculate standard deviation SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo); - + + if (pCtx->currentStage == REPEAT_SCAN && pStd->stage == 0) { + pStd->stage++; + avg_finalizer(pCtx); + + pResInfo->initialized = true; // set it initialized to avoid re-initialization + + // save average value into tmpBuf, for second stage scan + SAvgInfo *pAvg = GET_ROWCELL_INTERBUF(pResInfo); + + pStd->avg = GET_DOUBLE_VAL(pCtx->pOutput); + assert((isnan(pAvg->sum) && pAvg->num == 0) || (pStd->num == pAvg->num && pStd->avg == pAvg->sum)); + } + /* the first stage is to calculate average value */ if (pStd->stage == 0) { avg_function_f(pCtx, index); @@ -1574,7 +1621,7 @@ static void stddev_dst_function(SQLFunctionCtx *pCtx) { if (p == NULL) { return; } - + avg = p->avg; } @@ -1776,7 +1823,7 @@ static bool first_last_function_setup(SQLFunctionCtx *pCtx) { // todo opt for null block static void first_function(SQLFunctionCtx *pCtx) { - if (pCtx->order == TSDB_ORDER_DESC || pCtx->preAggVals.dataBlockLoaded == false) { + if (pCtx->order == TSDB_ORDER_DESC /*|| pCtx->preAggVals.dataBlockLoaded == false*/) { return; } @@ -1850,7 +1897,7 @@ static void first_dist_function(SQLFunctionCtx *pCtx) { * 1. data block that are not loaded * 2. scan data files in desc order */ - if (pCtx->order == TSDB_ORDER_DESC || pCtx->preAggVals.dataBlockLoaded == false) { + if (pCtx->order == TSDB_ORDER_DESC/* || pCtx->preAggVals.dataBlockLoaded == false*/) { return; } @@ -1921,7 +1968,7 @@ static void first_dist_func_merge(SQLFunctionCtx *pCtx) { * least one data in this block that is not null.(TODO opt for this case) */ static void last_function(SQLFunctionCtx *pCtx) { - if (pCtx->order != pCtx->param[0].i64 || pCtx->preAggVals.dataBlockLoaded == false) { + if (pCtx->order != pCtx->param[0].i64/* || pCtx->preAggVals.dataBlockLoaded == false*/) { return; } @@ -1934,6 +1981,7 @@ static void last_function(SQLFunctionCtx *pCtx) { continue; } } + memcpy(pCtx->pOutput, data, pCtx->inputBytes); TSKEY ts = GET_TS_DATA(pCtx, i); @@ -2013,13 +2061,7 @@ static void last_dist_function(SQLFunctionCtx *pCtx) { return; } - // data block is discard, not loaded, do not need to check it - if (!pCtx->preAggVals.dataBlockLoaded) { - return; - } - int32_t notNullElems = 0; - for (int32_t i = pCtx->size - 1; i >= 0; --i) { char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { @@ -2125,12 +2167,7 @@ static void last_row_finalizer(SQLFunctionCtx *pCtx) { // do nothing at the first stage SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult != DATA_SET_FLAG) { - if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(pCtx->pOutput, pCtx->outputType); - } else { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - } - + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); return; } @@ -2445,7 +2482,7 @@ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { } } -bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const char *minval, const char *maxval) { +bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const char *maxval) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo == NULL) { return true; @@ -2460,7 +2497,7 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const cha tValuePair **pRes = (tValuePair**) pTopBotInfo->res; - if (functionId == TSDB_FUNC_TOP) { + if (pCtx->functionId == TSDB_FUNC_TOP) { switch (pCtx->inputType) { case TSDB_DATA_TYPE_TINYINT: return GET_INT8_VAL(maxval) > pRes[0]->v.i64; @@ -2531,9 +2568,13 @@ static bool top_bottom_function_setup(SQLFunctionCtx *pCtx) { static void top_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - + STopBotInfo *pRes = getTopBotOutputInfo(pCtx); assert(pRes->num >= 0); + + if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].i64)) { + buildTopBotStruct(pRes, pCtx); + } for (int32_t i = 0; i < pCtx->size; ++i) { char *data = GET_INPUT_DATA(pCtx, i); @@ -2609,13 +2650,13 @@ static void bottom_function(SQLFunctionCtx *pCtx) { if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].i64)) { buildTopBotStruct(pRes, pCtx); } - + for (int32_t i = 0; i < pCtx->size; ++i) { char *data = GET_INPUT_DATA(pCtx, i); TSKEY ts = GET_TS_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; + continue; } notNullElems++; @@ -2648,7 +2689,7 @@ static void bottom_function_f(SQLFunctionCtx *pCtx, int32_t index) { if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].i64)) { buildTopBotStruct(pRes, pCtx); } - + SET_VAL(pCtx, 1, 1); do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64, pData, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); @@ -2729,6 +2770,17 @@ static void percentile_function(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); + if (pCtx->currentStage == REPEAT_SCAN && pInfo->stage == 0) { + // all data are null, set it completed + if (pInfo->numOfElems == 0) { + pResInfo->complete = true; + } else { + pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval); + } + + pInfo->stage += 1; + } + // the first stage, only acquire the min/max value if (pInfo->stage == 0) { if (pCtx->preAggVals.isSet) { @@ -2802,10 +2854,20 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { } SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo *pInfo = (SPercentileInfo *)GET_ROWCELL_INTERBUF(pResInfo); - if (pInfo->stage == 0) { + if (pCtx->currentStage == REPEAT_SCAN && pInfo->stage == 0) { + // all data are null, set it completed + if (pInfo->numOfElems == 0) { + pResInfo->complete = true; + } else { + pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval); + } + + pInfo->stage += 1; + } + + if (pInfo->stage == 0) { double v = 0; GET_TYPED_DATA(v, double, pCtx->inputType, pData); @@ -3240,8 +3302,6 @@ static void col_project_function(SQLFunctionCtx *pCtx) { pCtx->inputBytes); } } - - pCtx->pOutput += pCtx->size * pCtx->outputBytes; } static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { @@ -3547,9 +3607,6 @@ static void diff_function(SQLFunctionCtx *pCtx) { int32_t forwardStep = (isFirstBlock) ? notNullElems - 1 : notNullElems; GET_RES_INFO(pCtx)->numOfRes += forwardStep; - - pCtx->pOutput += forwardStep * pCtx->outputBytes; - pCtx->ptsOutputBuf = (char*)pCtx->ptsOutputBuf + forwardStep * TSDB_KEYSIZE; } } @@ -3631,7 +3688,7 @@ char *getArithColumnData(void *param, const char* name, int32_t colId) { } } - assert(index >= 0 && colId >= 0); + assert(index >= 0 /*&& colId >= 0*/); return pSupport->data[index] + pSupport->offset * pSupport->colList[index].bytes; } @@ -3640,7 +3697,6 @@ static void arithmetic_function(SQLFunctionCtx *pCtx) { SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; arithmeticTreeTraverse(sas->pArithExpr->pExpr, pCtx->size, pCtx->pOutput, sas, pCtx->order, getArithColumnData); - pCtx->pOutput += pCtx->outputBytes * pCtx->size; } static void arithmetic_function_f(SQLFunctionCtx *pCtx, int32_t index) { @@ -4171,50 +4227,88 @@ static void interp_function_impl(SQLFunctionCtx *pCtx) { } if (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP) { - *(TSKEY *) pCtx->pOutput = pCtx->startTs; + *(TSKEY *)pCtx->pOutput = pCtx->startTs; + } else if (type == TSDB_FILL_NULL) { + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); + } else if (type == TSDB_FILL_SET_VALUE) { + tVariantDump(&pCtx->param[1], pCtx->pOutput, pCtx->inputType, true); } else { - if (pCtx->start.key == INT64_MIN) { - assert(pCtx->end.key == INT64_MIN); - return; - } - - if (type == TSDB_FILL_NULL) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - } else if (type == TSDB_FILL_SET_VALUE) { - tVariantDump(&pCtx->param[1], pCtx->pOutput, pCtx->inputType, true); - } else if (type == TSDB_FILL_PREV) { - if (IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL) { - SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, pCtx->start.val); - } else { - assignVal(pCtx->pOutput, pCtx->start.ptr, pCtx->outputBytes, pCtx->inputType); + if (pCtx->start.key != INT64_MIN && pCtx->start.key < pCtx->startTs && pCtx->end.key > pCtx->startTs) { + if (type == TSDB_FILL_PREV) { + if (IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL) { + SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, pCtx->start.val); + } else { + assignVal(pCtx->pOutput, pCtx->start.ptr, pCtx->outputBytes, pCtx->inputType); + } + } else if (type == TSDB_FILL_NEXT) { + if (IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL) { + SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, pCtx->end.val); + } else { + assignVal(pCtx->pOutput, pCtx->end.ptr, pCtx->outputBytes, pCtx->inputType); + } + } else if (type == TSDB_FILL_LINEAR) { + SPoint point1 = {.key = pCtx->start.key, .val = &pCtx->start.val}; + SPoint point2 = {.key = pCtx->end.key, .val = &pCtx->end.val}; + SPoint point = {.key = pCtx->startTs, .val = pCtx->pOutput}; + + int32_t srcType = pCtx->inputType; + if (IS_NUMERIC_TYPE(srcType)) { // TODO should find the not null data? + if (isNull((char *)&pCtx->start.val, srcType) || isNull((char *)&pCtx->end.val, srcType)) { + setNull(pCtx->pOutput, srcType, pCtx->inputBytes); + } else { + taosGetLinearInterpolationVal(&point, pCtx->outputType, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); + } + } else { + setNull(pCtx->pOutput, srcType, pCtx->inputBytes); + } } - } else if (type == TSDB_FILL_NEXT) { - if (IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL) { - SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, pCtx->end.val); - } else { - assignVal(pCtx->pOutput, pCtx->end.ptr, pCtx->outputBytes, pCtx->inputType); + } else { + // no data generated yet + if (pCtx->size == 1) { + return; } - } else if (type == TSDB_FILL_LINEAR) { - SPoint point1 = {.key = pCtx->start.key, .val = &pCtx->start.val}; - SPoint point2 = {.key = pCtx->end.key, .val = &pCtx->end.val}; - SPoint point = {.key = pCtx->startTs, .val = pCtx->pOutput}; - - int32_t srcType = pCtx->inputType; - if (IS_NUMERIC_TYPE(srcType)) { // TODO should find the not null data? - if (isNull((char *)&pCtx->start.val, srcType) || isNull((char *)&pCtx->end.val, srcType)) { - setNull(pCtx->pOutput, srcType, pCtx->inputBytes); + + // check the timestamp in input buffer + TSKEY skey = GET_TS_DATA(pCtx, 0); + TSKEY ekey = GET_TS_DATA(pCtx, 1); + + // no data generated yet + if (!(skey < pCtx->startTs && ekey > pCtx->startTs)) { + return; + } + + assert(pCtx->start.key == INT64_MIN && skey < pCtx->startTs && ekey > pCtx->startTs); + + if (type == TSDB_FILL_PREV) { + assignVal(pCtx->pOutput, pCtx->pInput, pCtx->outputBytes, pCtx->inputType); + } else if (type == TSDB_FILL_NEXT) { + char* val = ((char*)pCtx->pInput) + pCtx->inputBytes; + assignVal(pCtx->pOutput, val, pCtx->outputBytes, pCtx->inputType); + } else if (type == TSDB_FILL_LINEAR) { + char *start = GET_INPUT_DATA(pCtx, 0); + char *end = GET_INPUT_DATA(pCtx, 1); + + SPoint point1 = {.key = skey, .val = start}; + SPoint point2 = {.key = ekey, .val = end}; + SPoint point = {.key = pCtx->startTs, .val = pCtx->pOutput}; + + int32_t srcType = pCtx->inputType; + if (IS_NUMERIC_TYPE(srcType)) { // TODO should find the not null data? + if (isNull(start, srcType) || isNull(end, srcType)) { + setNull(pCtx->pOutput, srcType, pCtx->inputBytes); + } else { + taosGetLinearInterpolationVal(&point, pCtx->outputType, &point1, &point2, srcType); + } } else { - taosGetLinearInterpolationVal(&point, pCtx->outputType, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); + setNull(pCtx->pOutput, srcType, pCtx->inputBytes); } - } else { - setNull(pCtx->pOutput, srcType, pCtx->inputBytes); } } } SET_VAL(pCtx, 1, 1); - } + static void interp_function(SQLFunctionCtx *pCtx) { // at this point, the value is existed, return directly if (pCtx->size > 0) { @@ -4289,11 +4383,22 @@ static void ts_comp_finalize(SQLFunctionCtx *pCtx) { STSBuf * pTSbuf = pInfo->pTSBuf; tsBufFlush(pTSbuf); - + qDebug("total timestamp :%"PRId64, pTSbuf->numOfTotal); + + // TODO refactor transfer ownership of current file *(FILE **)pCtx->pOutput = pTSbuf->f; + pResInfo->complete = true; + + // get the file size + struct stat fStat; + if ((fstat(fileno(pTSbuf->f), &fStat) == 0)) { + pResInfo->numOfRes = fStat.st_size; + } + pTSbuf->remainOpen = true; tsBufDestroy(pTSbuf); + doFinalizer(pCtx); } @@ -4637,10 +4742,126 @@ static void sumrate_finalizer(SQLFunctionCtx *pCtx) { doFinalizer(pCtx); } +void blockInfo_func(SQLFunctionCtx* pCtx) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); -///////////////////////////////////////////////////////////////////////////////////////////// + int32_t len = *(int32_t*) pCtx->pInput; + blockDistInfoFromBinary((char*)pCtx->pInput + sizeof(int32_t), len, pDist); + pDist->rowSize = (int16_t) pCtx->param[0].i64; + + memcpy(pCtx->pOutput, pCtx->pInput, sizeof(int32_t) + len); + pResInfo->numOfRes = 1; + pResInfo->hasResult = DATA_SET_FLAG; +} +static void mergeTableBlockDist(STableBlockDist* pDist, const STableBlockDist* pSrc) { + assert(pDist != NULL && pSrc != NULL); + pDist->numOfTables += pSrc->numOfTables; + pDist->numOfRowsInMemTable += pSrc->numOfRowsInMemTable; + pDist->numOfFiles += pSrc->numOfFiles; + pDist->totalSize += pSrc->totalSize; + + if (pDist->dataBlockInfos == NULL) { + pDist->dataBlockInfos = taosArrayInit(4, sizeof(SFileBlockInfo)); + } + + taosArrayPushBatch(pDist->dataBlockInfos, pSrc->dataBlockInfos->pData, (int32_t) taosArrayGetSize(pSrc->dataBlockInfos)); +} + +void block_func_merge(SQLFunctionCtx* pCtx) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + + STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); + STableBlockDist info = {0}; + + int32_t len = *(int32_t*) pCtx->pInput; + blockDistInfoFromBinary(((char*)pCtx->pInput) + sizeof(int32_t), len, &info); + + mergeTableBlockDist(pDist, &info); +} + +static int32_t doGetPercentile(const SArray* pArray, double rate) { + int32_t len = (int32_t)taosArrayGetSize(pArray); + if (len <= 0) { + return 0; + } + + assert(rate >= 0 && rate <= 1.0); + int idx = (int32_t)((len - 1) * rate); + + return ((SFileBlockInfo *)(taosArrayGet(pArray, idx)))->numOfRows; +} + +static int compareBlockInfo(const void *pLeft, const void *pRight) { + int32_t left = ((SFileBlockInfo *)pLeft)->numOfRows; + int32_t right = ((SFileBlockInfo *)pRight)->numOfRows; + + if (left > right) return 1; + if (left < right) return -1; + return 0; +} + +void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) { + if (pTableBlockDist == NULL) { + return; + } + + int64_t min = INT64_MAX, max = INT64_MIN, avg = 0; + SArray* blockInfos= pTableBlockDist->dataBlockInfos; + int64_t totalRows = 0, totalBlocks = taosArrayGetSize(blockInfos); + + for (size_t i = 0; i < taosArrayGetSize(blockInfos); i++) { + SFileBlockInfo *blockInfo = taosArrayGet(blockInfos, i); + int64_t rows = blockInfo->numOfRows; + + min = MIN(min, rows); + max = MAX(max, rows); + totalRows += rows; + } + + avg = totalBlocks > 0 ? (int64_t)(totalRows/totalBlocks) : 0; + taosArraySort(blockInfos, compareBlockInfo); + + uint64_t totalLen = pTableBlockDist->totalSize; + int32_t rowSize = pTableBlockDist->rowSize; + + int sz = sprintf(result + VARSTR_HEADER_SIZE, + "summary: \n\t " + "5th=[%d], 10th=[%d], 20th=[%d], 30th=[%d], 40th=[%d], 50th=[%d]\n\t " + "60th=[%d], 70th=[%d], 80th=[%d], 90th=[%d], 95th=[%d], 99th=[%d]\n\t " + "Min=[%"PRId64"(Rows)] Max=[%"PRId64"(Rows)] Avg=[%"PRId64"(Rows)] Stddev=[%.2f] \n\t " + "Rows=[%"PRId64"], Blocks=[%"PRId64"], Size=[%.3f(Kb)] Comp=[%.2f%%]\n\t " + "RowsInMem=[%d] \n\t SeekHeaderTime=[%d(us)]", + doGetPercentile(blockInfos, 0.05), doGetPercentile(blockInfos, 0.10), + doGetPercentile(blockInfos, 0.20), doGetPercentile(blockInfos, 0.30), + doGetPercentile(blockInfos, 0.40), doGetPercentile(blockInfos, 0.50), + doGetPercentile(blockInfos, 0.60), doGetPercentile(blockInfos, 0.70), + doGetPercentile(blockInfos, 0.80), doGetPercentile(blockInfos, 0.90), + doGetPercentile(blockInfos, 0.95), doGetPercentile(blockInfos, 0.99), + min, max, avg, 0.0, + totalRows, totalBlocks, totalLen/1024.0, (double)(totalLen*100.0)/(rowSize*totalRows), + pTableBlockDist->numOfRowsInMemTable, pTableBlockDist->firstSeekTimeUs); + varDataSetLen(result, sz); + UNUSED(sz); +} + +void blockinfo_func_finalizer(SQLFunctionCtx* pCtx) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); + + pDist->rowSize = (int16_t)pCtx->param[0].i64; + generateBlockDistResult(pDist, pCtx->pOutput); + + // cannot set the numOfIteratedElems again since it is set during previous iteration + pResInfo->numOfRes = 1; + pResInfo->hasResult = DATA_SET_FLAG; + + doFinalizer(pCtx); +} + +///////////////////////////////////////////////////////////////////////////////////////////// /* * function compatible list. * tag and ts are not involved in the compatibility check @@ -4659,8 +4880,8 @@ int32_t functionCompatList[] = { 4, -1, -1, 1, 1, 1, 1, 1, 1, -1, // tag, colprj, tagprj, arithmetic, diff, first_dist, last_dist, interp rate irate 1, 1, 1, 1, -1, 1, 1, 5, 1, 1, - // sum_rate, sum_irate, avg_rate, avg_irate - 1, 1, 1, 1, + // sum_rate, sum_irate, avg_rate, avg_irate, tid_tag, blk_info + 1, 1, 1, 1, 6, 7 }; SAggFunctionInfo aAggs[] = {{ @@ -4675,7 +4896,7 @@ SAggFunctionInfo aAggs[] = {{ no_next_step, doFinalizer, count_func_merge, - count_load_data_info, + countRequired, }, { // 1 @@ -4860,7 +5081,7 @@ SAggFunctionInfo aAggs[] = {{ no_next_step, spread_function_finalizer, spread_func_merge, - count_load_data_info, + countRequired, }, { // 14 @@ -4902,7 +5123,7 @@ SAggFunctionInfo aAggs[] = {{ no_next_step, doFinalizer, copy_function, - no_data_info, + noDataRequired, }, { // 17 @@ -4930,7 +5151,7 @@ SAggFunctionInfo aAggs[] = {{ no_next_step, doFinalizer, copy_function, - no_data_info, + noDataRequired, }, { // 19 @@ -4958,7 +5179,7 @@ SAggFunctionInfo aAggs[] = {{ no_next_step, doFinalizer, copy_function, - no_data_info, + noDataRequired, }, { // 21, column project sql function @@ -4986,7 +5207,7 @@ SAggFunctionInfo aAggs[] = {{ no_next_step, doFinalizer, copy_function, - no_data_info, + noDataRequired, }, { // 23 @@ -5159,7 +5380,7 @@ SAggFunctionInfo aAggs[] = {{ }, { // 35 - "tid_tag", // return table id and the corresponding tags for join match and subscribe + "tbid", // return table id and the corresponding tags for join match and subscribe TSDB_FUNC_TID_TAG, TSDB_FUNC_TID_TAG, TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STABLE, @@ -5170,4 +5391,18 @@ SAggFunctionInfo aAggs[] = {{ noop1, noop1, dataBlockRequired, - } }; + }, + { + // 35 + "_block_dist", // return table id and the corresponding tags for join match and subscribe + TSDB_FUNC_BLKINFO, + TSDB_FUNC_BLKINFO, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STABLE, + function_setup, + blockInfo_func, + noop2, + no_next_step, + blockinfo_func_finalizer, + block_func_merge, + dataBlockRequired, + }}; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 7b0dbdc5c225acb436e65dbc54e57bfad077c33e..4b3cd86c0799a2cf875efe0e475416333fdce432 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -26,24 +26,22 @@ #include "queryLog.h" #include "tlosertree.h" #include "ttype.h" - -#define MAX_ROWS_PER_RESBUF_PAGE ((1u<<12) - 1) - -/** - * check if the primary column is load by default, otherwise, the program will - * forced to load primary column explicitly. - */ +#include "tscompression.h" #define IS_MASTER_SCAN(runtime) ((runtime)->scanFlag == MASTER_SCAN) #define IS_REVERSE_SCAN(runtime) ((runtime)->scanFlag == REVERSE_SCAN) #define SET_MASTER_SCAN_FLAG(runtime) ((runtime)->scanFlag = MASTER_SCAN) #define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN) -#define GET_QINFO_ADDR(x) ((SQInfo *)((char *)(x)-offsetof(SQInfo, runtimeEnv))) - -#define GET_COL_DATA_POS(query, index, step) ((query)->pos + (index) * (step)) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) +#define CHECK_IF_QUERY_KILLED(_q) \ + do { \ + if (isQueryKilled((_q)->qinfo)) { \ + longjmp((_q)->env, TSDB_CODE_TSC_QUERY_CANCELLED); \ + } \ + } while (0) + #define SDATA_BLOCK_INITIALIZER (SDataBlockInfo) {{0}, 0} #define TIME_WINDOW_COPY(_dst, _src) do {\ @@ -57,20 +55,10 @@ enum { TS_JOIN_TAG_NOT_EQUALS = 2, }; -typedef struct { - int32_t status; // query status - TSKEY lastKey; // the lastKey value before query executed - STimeWindow w; // whole query time window - int32_t windowIndex; // index of active time window result for interval query - STSCursor cur; -} SQueryStatusInfo; - -typedef struct { - SArray *dataBlockInfos; - int64_t firstSeekTimeUs; - int64_t numOfRowsInMemTable; - char *result; -} STableBlockDist; +typedef enum SResultTsInterpType { + RESULT_ROW_START_INTERP = 1, + RESULT_ROW_END_INTERP = 2, +} SResultTsInterpType; #if 0 static UNUSED_FUNC void *u_malloc (size_t __size) { @@ -110,11 +98,8 @@ static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { #define GET_NUM_OF_TABLEGROUP(q) taosArrayGetSize((q)->tableqinfoGroupInfo.pGroupList) #define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->interval.interval > 0) -static void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv); - uint64_t queryHandleId = 0; - - +` int32_t getMaximumIdleDurationSec() { return tsShellActivityTimer * 2; } @@ -156,84 +141,123 @@ static void getNextTimeWindow(SQuery* pQuery, STimeWindow* tw) { tw->ekey -= 1; } -static void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult); -static void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult); +static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes); +static void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, + int32_t numOfCols, int32_t* rowCellInfoOffset); + +void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId); -static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo, - SDataStatis *pStatis, SExprInfo* pExprInfo); +static void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColIndex* pColIndex); -static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo); -static void resetDefaultResInfoOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static bool hasMainOutput(SQuery *pQuery); -static int32_t setTimestampListJoinInfo(SQInfo *pQInfo, STableQueryInfo *pTableQueryInfo); +static int32_t setTimestampListJoinInfo(SQueryRuntimeEnv* pRuntimeEnv, tVariant* pTag, STableQueryInfo *pTableQueryInfo); static void releaseQueryBuf(size_t numOfTables); static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order); -static void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type); static STsdbQueryCond createTsdbQueryCond(SQuery* pQuery, STimeWindow* win); -static STableIdInfo createTableIdInfo(SQuery* pQuery); - -bool doFilterData(SQuery *pQuery, int32_t elemPos) { - for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { - SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; - - char *pElem = (char*)pFilterInfo->pData + pFilterInfo->info.bytes * elemPos; +static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo); + +static void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInfo* pDownstream); + +static int32_t getNumOfScanTimes(SQuery* pQuery); +static bool isFixedOutputQuery(SQuery* pQuery); + +static SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime); +static SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime); +static SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv); + +static SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +static SOperatorInfo* createArithOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +static SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream); +static SOperatorInfo* createOffsetOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream); +static SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +static SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +static SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +static SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +static SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +static SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +static SOperatorInfo* createTagScanOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput); +static SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv); + +static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); +static void destroySFillOperatorInfo(void* param, int32_t numOfOutput); +static void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput); +static void destroyArithOperatorInfo(void* param, int32_t numOfOutput); +static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput); +static void destroyOperatorInfo(SOperatorInfo* pOperator); + +static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock); + +static int32_t getGroupbyColumnIndex(SSqlGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock); +static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SGroupbyOperatorInfo *pInfo, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex); + +static void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size); +static void getAlignQueryTimeWindow(SQuery *pQuery, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win); +static bool isPointInterpoQuery(SQuery *pQuery); +static void setResultBufSize(SQuery* pQuery, SRspResultInfo* pResultInfo); +static void setCtxTagForJoin(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); +static void setParamForStableStddev(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr); +static void doSetTableGroupOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, + SQLFunctionCtx* pCtx, int32_t* rowCellInfoOffset, int32_t numOfOutput, + int32_t groupIndex); + +// setup the output buffer for each operator +static SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows) { + const static int32_t minSize = 8; + + SSDataBlock *res = calloc(1, sizeof(SSDataBlock)); + res->info.numOfCols = numOfOutput; + + res->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); + for (int32_t i = 0; i < numOfOutput; ++i) { + SColumnInfoData idata = {{0}}; + idata.info.type = pExpr[i].type; + idata.info.bytes = pExpr[i].bytes; + idata.info.colId = pExpr[i].base.resColId; - bool qualified = false; - for (int32_t j = 0; j < pFilterInfo->numOfFilters; ++j) { - SColumnFilterElem *pFilterElem = &pFilterInfo->pFilters[j]; + idata.pData = calloc(1, MAX(idata.info.bytes * numOfRows, minSize)); // at least to hold a pointer on x64 platform + taosArrayPush(res->pDataBlock, &idata); + } - bool isnull = isNull(pElem, pFilterInfo->info.type); - if (isnull) { - if (pFilterElem->fp == isNullOperator) { - qualified = true; - break; - } else { - continue; - } - } else { - if (pFilterElem->fp == notNullOperator) { - qualified = true; - break; - } else if (pFilterElem->fp == isNullOperator) { - continue; - } - } + return res; +} - if (pFilterElem->fp(pFilterElem, pElem, pElem, pFilterInfo->info.type)) { - qualified = true; - break; - } - } +static void* destroyOutputBuf(SSDataBlock* pBlock) { + if (pBlock == NULL) { + return NULL; + } - if (!qualified) { - return false; - } + int32_t numOfOutput = pBlock->info.numOfCols; + for(int32_t i = 0; i < numOfOutput; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + tfree(pColInfoData->pData); } - return true; + taosArrayDestroy(pBlock->pDataBlock); + tfree(pBlock->pBlockStatis); + tfree(pBlock); + return NULL; } -int64_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv) { +int32_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput) { SQuery *pQuery = pRuntimeEnv->pQuery; bool hasMainFunction = hasMainOutput(pQuery); - int64_t maxOutput = 0; - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int32_t functionId = pQuery->pExpr1[j].base.functionId; + int32_t maxOutput = 0; + for (int32_t j = 0; j < numOfOutput; ++j) { + int32_t id = pCtx[j].functionId; /* * ts, tag, tagprj function can not decide the output number of current query * the number of output result is decided by main output */ - if (hasMainFunction && - (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ)) { + if (hasMainFunction && (id == TSDB_FUNC_TS || id == TSDB_FUNC_TAG || id == TSDB_FUNC_TAGPRJ)) { continue; } - SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + SResultRowCellInfo *pResInfo = GET_RES_INFO(&pCtx[j]); if (pResInfo != NULL && maxOutput < pResInfo->numOfRes) { maxOutput = pResInfo->numOfRes; } @@ -243,27 +267,14 @@ int64_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv) { return maxOutput; } -/* - * the value of number of result needs to be update due to offset value upated. - */ -void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); - - int16_t functionId = pRuntimeEnv->pCtx[j].functionId; - if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ || - functionId == TSDB_FUNC_TS_DUMMY) { - continue; - } - - assert(pResInfo->numOfRes > numOfRes); - pResInfo->numOfRes = numOfRes; +static void clearNumOfRes(SQLFunctionCtx* pCtx, int32_t numOfOutput) { + for (int32_t j = 0; j < numOfOutput; ++j) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(&pCtx[j]); + pResInfo->numOfRes = 0; } } -bool isGroupbyColumn(SSqlGroupbyExpr *pGroupbyExpr) { +static bool isGroupbyColumn(SSqlGroupbyExpr *pGroupbyExpr) { if (pGroupbyExpr == NULL || pGroupbyExpr->numOfGroupCols == 0) { return false; } @@ -283,7 +294,7 @@ bool isGroupbyColumn(SSqlGroupbyExpr *pGroupbyExpr) { return false; } -bool isStabledev(SQuery* pQuery) { +static bool isStabledev(SQuery* pQuery) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functId = pQuery->pExpr1[i].base.functionId; if (functId == TSDB_FUNC_STDDEV_DST) { @@ -294,36 +305,12 @@ bool isStabledev(SQuery* pQuery) { return false; } -int16_t getGroupbyColumnType(SQuery *pQuery, SSqlGroupbyExpr *pGroupbyExpr) { - assert(pGroupbyExpr != NULL); - - int32_t colId = -2; - int16_t type = TSDB_DATA_TYPE_NULL; - - for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { - SColIndex *pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, i); - if (TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { - colId = pColIndex->colId; - break; - } - } - - for (int32_t i = 0; i < pQuery->numOfCols; ++i) { - if (colId == pQuery->colList[i].colId) { - type = pQuery->colList[i].type; - break; - } - } - - return type; -} - -static bool isSelectivityWithTagsQuery(SQuery *pQuery) { +static bool isSelectivityWithTagsQuery(SQLFunctionCtx *pCtx, int32_t numOfOutput) { bool hasTags = false; int32_t numOfSelectivity = 0; - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functId = pQuery->pExpr1[i].base.functionId; + for (int32_t i = 0; i < numOfOutput; ++i) { + int32_t functId = pCtx[i].functionId; if (functId == TSDB_FUNC_TAG_DUMMY || functId == TSDB_FUNC_TS_DUMMY) { hasTags = true; continue; @@ -334,11 +321,7 @@ static bool isSelectivityWithTagsQuery(SQuery *pQuery) { } } - if (numOfSelectivity > 0 && hasTags) { - return true; - } - - return false; + return (numOfSelectivity > 0 && hasTags); } static bool isProjQuery(SQuery *pQuery) { @@ -354,20 +337,6 @@ static bool isProjQuery(SQuery *pQuery) { static bool isTsCompQuery(SQuery *pQuery) { return pQuery->pExpr1[0].base.functionId == TSDB_FUNC_TS_COMP; } -static bool limitOperator(SQuery* pQuery, void* qinfo) { - if ((pQuery->limit.limit > 0) && (pQuery->rec.total + pQuery->rec.rows > pQuery->limit.limit)) { - pQuery->rec.rows = pQuery->limit.limit - pQuery->rec.total; - - qDebug("QInfo:%p discard remain data due to result limitation, limit:%"PRId64", current return:%" PRId64 ", total:%"PRId64, - qinfo, pQuery->limit.limit, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); - assert(pQuery->rec.rows >= 0); - setQueryStatus(pQuery, QUERY_COMPLETED); - return true; - } - - return false; -} - static bool isTopBottomQuery(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pExpr1[i].base.functionId; @@ -412,33 +381,44 @@ static bool hasTagValOutput(SQuery* pQuery) { return false; } -/** - * @param pQuery - * @param col - * @param pDataBlockInfo - * @param pStatis - * @param pColStatis - * @return - */ -static bool hasNullValue(SColIndex* pColIndex, SDataStatis *pStatis, SDataStatis **pColStatis) { - if (pStatis != NULL && TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { - *pColStatis = &pStatis[pColIndex->colIndex]; - assert((*pColStatis)->colId == pColIndex->colId); - } else { - *pColStatis = NULL; - } - +static bool hasNullRv(SColIndex* pColIndex, SDataStatis *pStatis) { if (TSDB_COL_IS_TAG(pColIndex->flag) || TSDB_COL_IS_UD_COL(pColIndex->flag) || pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { return false; } - if ((*pColStatis) != NULL && (*pColStatis)->numOfNull == 0) { + if (pStatis != NULL && pStatis->numOfNull == 0) { return false; } return true; } +static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, SQueryRuntimeEnv* pRuntimeEnv) { + // more than the capacity, reallocate the resources + if (pResultRowInfo->size < pResultRowInfo->capacity) { + return; + } + + int64_t newCapacity = 0; + if (pResultRowInfo->capacity > 10000) { + newCapacity = (int64_t)(pResultRowInfo->capacity * 1.25); + } else { + newCapacity = (int64_t)(pResultRowInfo->capacity * 1.5); + } + + char *t = realloc(pResultRowInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); + if (t == NULL) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + pResultRowInfo->pResult = (SResultRow **)t; + + int32_t inc = (int32_t)newCapacity - pResultRowInfo->capacity; + memset(&pResultRowInfo->pResult[pResultRowInfo->capacity], 0, POINTER_BYTES * inc); + + pResultRowInfo->capacity = (int32_t)newCapacity; +} + static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, char *pData, int16_t bytes, bool masterscan, uint64_t uid) { bool existed = false; @@ -469,28 +449,7 @@ static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SRes } if (!existed) { - // TODO refactor - // more than the capacity, reallocate the resources - if (pResultRowInfo->size >= pResultRowInfo->capacity) { - int64_t newCapacity = 0; - if (pResultRowInfo->capacity > 10000) { - newCapacity = (int64_t)(pResultRowInfo->capacity * 1.25); - } else { - newCapacity = (int64_t)(pResultRowInfo->capacity * 1.5); - } - - char *t = realloc(pResultRowInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); - if (t == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - pResultRowInfo->pResult = (SResultRow **)t; - - int32_t inc = (int32_t)newCapacity - pResultRowInfo->capacity; - memset(&pResultRowInfo->pResult[pResultRowInfo->capacity], 0, POINTER_BYTES * inc); - - pResultRowInfo->capacity = (int32_t)newCapacity; - } + prepareResultListBuffer(pResultRowInfo, pRuntimeEnv); SResultRow *pResult = NULL; @@ -520,19 +479,30 @@ static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SRes } // get the correct time window according to the handled timestamp -static STimeWindow getActiveTimeWindow(SResultRowInfo *pWindowResInfo, int64_t ts, SQuery *pQuery) { +static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t ts, SQuery *pQuery) { STimeWindow w = {0}; - if (pWindowResInfo->curIndex == -1) { // the first window, from the previous stored value - w.skey = pWindowResInfo->prevSKey; + if (pResultRowInfo->curIndex == -1) { // the first window, from the previous stored value + if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { + if (QUERY_IS_ASC_QUERY(pQuery)) { + getAlignQueryTimeWindow(pQuery, ts, ts, pQuery->window.ekey, &w); + } else { // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp + getAlignQueryTimeWindow(pQuery, ts, pQuery->window.ekey, ts, &w); + } + + pResultRowInfo->prevSKey = w.skey; + } else { + w.skey = pResultRowInfo->prevSKey; + } + if (pQuery->interval.intervalUnit == 'n' || pQuery->interval.intervalUnit == 'y') { w.ekey = taosTimeAdd(w.skey, pQuery->interval.interval, pQuery->interval.intervalUnit, pQuery->precision) - 1; } else { w.ekey = w.skey + pQuery->interval.interval - 1; } } else { - int32_t slot = curTimeWindowIndex(pWindowResInfo); - SResultRow* pWindowRes = getResultRow(pWindowResInfo, slot); + int32_t slot = curTimeWindowIndex(pResultRowInfo); + SResultRow* pWindowRes = getResultRow(pResultRowInfo, slot); w = pWindowRes->win; } @@ -568,8 +538,8 @@ static STimeWindow getActiveTimeWindow(SResultRowInfo *pWindowResInfo, int64_t t return w; } -static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t tid, - int32_t numOfRowsPerPage) { +// a new buffer page for each table. Needs to opt this design +static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t tid, uint32_t size) { if (pWindowRes->pageId != -1) { return 0; } @@ -587,7 +557,7 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf pData = getResBufPage(pResultBuf, pi->pageId); pageId = pi->pageId; - if (pData->num >= numOfRowsPerPage) { + if (pData->num + size > pResultBuf->pageSize) { // release current page first, and prepare the next one releaseResBufPageInfo(pResultBuf, pi); pData = getNewDataBuf(pResultBuf, tid, &pageId); @@ -604,8 +574,9 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf // set the number of rows in current disk page if (pWindowRes->pageId == -1) { // not allocated yet, allocate new buffer pWindowRes->pageId = pageId; - pWindowRes->rowId = (int32_t)(pData->num++); + pWindowRes->offset = (int32_t)pData->num; + pData->num += size; assert(pWindowRes->pageId >= 0); } @@ -613,7 +584,8 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf } static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, STimeWindow *win, - bool masterscan, SResultRow **pResult, int64_t groupId) { + bool masterscan, SResultRow **pResult, int64_t groupId, SQLFunctionCtx* pCtx, + int32_t numOfOutput, int32_t* rowCellInfoOffset) { assert(win->skey <= win->ekey); SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; @@ -625,7 +597,7 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRow // not assign result buffer yet, add new result buffer if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, (int32_t) groupId, pRuntimeEnv->numOfRowsPerPage); + int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, (int32_t) groupId, pRuntimeEnv->pQuery->intermediateResultRowSize); if (ret != TSDB_CODE_SUCCESS) { return -1; } @@ -634,21 +606,11 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRow // set time window for current result pResultRow->win = (*win); *pResult = pResultRow; - setResultRowOutputBufInitCtx(pRuntimeEnv, pResultRow); + setResultRowOutputBufInitCtx(pRuntimeEnv, pResultRow, pCtx, numOfOutput, rowCellInfoOffset); return TSDB_CODE_SUCCESS; } -static bool getResultRowStatus(SResultRowInfo *pWindowResInfo, int32_t slot) { - assert(slot >= 0 && slot < pWindowResInfo->size); - return pWindowResInfo->pResult[slot]->closed; -} - -typedef enum SResultTsInterpType { - RESULT_ROW_START_INTERP = 1, - RESULT_ROW_END_INTERP = 2, -} SResultTsInterpType; - static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); if (type == RESULT_ROW_START_INTERP) { @@ -746,13 +708,14 @@ static void doUpdateResultRowIndex(SResultRowInfo*pResultRowInfo, TSKEY lastKey, } } -static void updateResultRowIndex(SResultRowInfo* pResultRowInfo, STableQueryInfo* pTableQueryInfo, bool ascQuery, bool timeWindowInterpo) { - if ((pTableQueryInfo->lastKey > pTableQueryInfo->win.ekey && ascQuery) || (pTableQueryInfo->lastKey < pTableQueryInfo->win.ekey && (!ascQuery))) { +static void updateResultRowInfoActiveIndex(SResultRowInfo* pResultRowInfo, SQuery* pQuery, TSKEY lastKey) { + bool ascQuery = QUERY_IS_ASC_QUERY(pQuery); + if ((lastKey > pQuery->window.ekey && ascQuery) || (lastKey < pQuery->window.ekey && (!ascQuery))) { closeAllResultRows(pResultRowInfo); pResultRowInfo->curIndex = pResultRowInfo->size - 1; } else { - int32_t step = ascQuery? 1:-1; - doUpdateResultRowIndex(pResultRowInfo, pTableQueryInfo->lastKey - step, ascQuery, timeWindowInterpo); + int32_t step = ascQuery ? 1 : -1; + doUpdateResultRowIndex(pResultRowInfo, lastKey - step, ascQuery, pQuery->timeWindowInterpo); } } @@ -796,31 +759,27 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo return num; } -static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int32_t col, int32_t size, SArray *pDataBlock); - -static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow *pWin, int32_t offset, - int32_t forwardStep, TSKEY *tsCol, int32_t numOfTotal, SArray *pDataBlock) { - SQuery * pQuery = pRuntimeEnv->pQuery; - SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - +static void doApplyFunctions(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset, + int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput) { + SQuery *pQuery = pRuntimeEnv->pQuery; bool hasPrev = pCtx[0].preAggVals.isSet; - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { + for (int32_t k = 0; k < numOfOutput; ++k) { pCtx[k].size = forwardStep; pCtx[k].startTs = pWin->skey; - char *dataBlock = getDataBlock(pRuntimeEnv, &pRuntimeEnv->sasArray[k], k, numOfTotal, pDataBlock); + char* start = pCtx[k].pInput; int32_t pos = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1); - if (dataBlock != NULL) { - pCtx[k].pInput = (char *)dataBlock + pos * pCtx[k].inputBytes; + if (pCtx[k].pInput != NULL) { + pCtx[k].pInput = (char *)pCtx[k].pInput + pos * pCtx[k].inputBytes; } if (tsCol != NULL) { pCtx[k].ptsList = &tsCol[pos]; } - int32_t functionId = pQuery->pExpr1[k].base.functionId; + int32_t functionId = pCtx[k].functionId; // not a whole block involved in query processing, statistics data can not be used // NOTE: the original value of isSet have been changed here @@ -834,27 +793,13 @@ static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow // restore it pCtx[k].preAggVals.isSet = hasPrev; + pCtx[k].pInput = start; } } -static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow *pWin, int32_t offset) { - SQuery *pQuery = pRuntimeEnv->pQuery; - SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - pCtx[k].startTs = pWin->skey; - - int32_t functionId = pQuery->pExpr1[k].base.functionId; - if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { - aAggs[functionId].xFunctionF(&pCtx[k], offset); - } - } -} -static int32_t getNextQualifiedWindow(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow *pNext, SDataBlockInfo *pDataBlockInfo, +static int32_t getNextQualifiedWindow(SQuery* pQuery, STimeWindow *pNext, SDataBlockInfo *pDataBlockInfo, TSKEY *primaryKeys, __block_search_fn_t searchFn, int32_t prevPosition) { - SQuery *pQuery = pRuntimeEnv->pQuery; - getNextTimeWindow(pQuery, pNext); // next time window is not in current block @@ -944,126 +889,251 @@ static FORCE_INLINE TSKEY reviseWindowEkey(SQuery *pQuery, STimeWindow *pWindow) return ekey; } -//todo binary search -static void* getDataBlockImpl(SArray* pDataBlock, int32_t colId) { - int32_t numOfCols = (int32_t)taosArrayGetSize(pDataBlock); - - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData *p = taosArrayGet(pDataBlock, i); - if (colId == p->info.colId) { - return p->pData; +static void setNotInterpoWindowKey(SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) { + if (type == RESULT_ROW_START_INTERP) { + for (int32_t k = 0; k < numOfOutput; ++k) { + pCtx[k].start.key = INT64_MIN; + } + } else { + for (int32_t k = 0; k < numOfOutput; ++k) { + pCtx[k].end.key = INT64_MIN; } } - - return NULL; } -// todo refactor -static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int32_t col, int32_t size, SArray *pDataBlock) { +// window start key interpolation + + +static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray* pDataBlock, + int32_t rowIndex) { if (pDataBlock == NULL) { - return NULL; + return; } - char *dataBlock = NULL; - SQuery *pQuery = pRuntimeEnv->pQuery; + SQuery* pQuery = pRuntimeEnv->pQuery; + for (int32_t k = 0; k < pQuery->numOfCols; ++k) { + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k); + memcpy(pRuntimeEnv->prevRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes); + } +} - int32_t functionId = pQuery->pExpr1[col].base.functionId; - if (functionId == TSDB_FUNC_ARITHM) { - sas->pArithExpr = &pQuery->pExpr1[col]; - sas->offset = (QUERY_IS_ASC_QUERY(pQuery))? pQuery->pos : pQuery->pos - (size - 1); - sas->colList = pQuery->colList; - sas->numOfCols = pQuery->numOfCols; - - // here the pQuery->colList and sas->colList are identical - int32_t numOfCols = (int32_t)taosArrayGetSize(pDataBlock); - for (int32_t i = 0; i < pQuery->numOfCols; ++i) { - SColumnInfo *pColMsg = &pQuery->colList[i]; - - dataBlock = NULL; - for (int32_t k = 0; k < numOfCols; ++k) { //todo refactor - SColumnInfoData *p = taosArrayGet(pDataBlock, k); - if (pColMsg->colId == p->info.colId) { - dataBlock = p->pData; - break; - } - } +static TSKEY getStartTsKey(SQuery* pQuery, STimeWindow* win, const TSKEY* tsCols, int32_t rows) { + TSKEY ts = TSKEY_INITIAL_VAL; + + bool ascQuery = QUERY_IS_ASC_QUERY(pQuery); + if (tsCols == NULL) { + ts = ascQuery? win->skey : win->ekey; + } else { + int32_t offset = ascQuery? 0:rows-1; + ts = tsCols[offset]; + } + + return ts; +} + +static void setArithParams(SArithmeticSupport* sas, SExprInfo *pExprInfo, SSDataBlock* pSDataBlock) { + sas->numOfCols = (int32_t) pSDataBlock->info.numOfCols; + sas->pArithExpr = pExprInfo; + + sas->colList = calloc(1, pSDataBlock->info.numOfCols*sizeof(SColumnInfo)); + for(int32_t i = 0; i < sas->numOfCols; ++i) { + SColumnInfoData* pColData = taosArrayGet(pSDataBlock->pDataBlock, i); + sas->colList[i] = pColData->info; + } + + sas->data = calloc(sas->numOfCols, POINTER_BYTES); + + // set the input column data + for (int32_t f = 0; f < pSDataBlock->info.numOfCols; ++f) { + SColumnInfoData *pColumnInfoData = taosArrayGet(pSDataBlock->pDataBlock, f); + sas->data[f] = pColumnInfoData->pData; + } +} - assert(dataBlock != NULL); - sas->data[i] = dataBlock; // start from the offset +static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); +static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { + for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { + pCtx[i].order = order; + pCtx[i].size = pBlock->info.rows; + pCtx[i].currentStage = (uint8_t)pOperator->pRuntimeEnv->scanFlag; + + setBlockStatisInfo(&pCtx[i], pBlock, &pOperator->pExpr[i].base.colInfo); + } +} + +static void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { + if (pCtx[0].functionId == TSDB_FUNC_ARITHM) { + SArithmeticSupport* pSupport = (SArithmeticSupport*) pCtx[0].param[1].pz; + if (pSupport->colList == NULL) { + doSetInputDataBlock(pOperator, pCtx, pBlock, order); + } else { + doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order); + } + } else { + if (pCtx[0].pInput == NULL && pBlock->pDataBlock != NULL) { + doSetInputDataBlock(pOperator, pCtx, pBlock, order); + } else { + doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order); } + } +} - } else { // other type of query function - SColIndex *pCol = &pQuery->pExpr1[col].base.colInfo; - if (TSDB_COL_IS_NORMAL_COL(pCol->flag)) { - SColIndex* pColIndex = &pQuery->pExpr1[col].base.colInfo; - SColumnInfoData *p = taosArrayGet(pDataBlock, pColIndex->colIndex); - assert(p->info.colId == pColIndex->colId); +static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { + for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { + pCtx[i].order = order; + pCtx[i].size = pBlock->info.rows; + pCtx[i].currentStage = (uint8_t)pOperator->pRuntimeEnv->scanFlag; - dataBlock = p->pData; + setBlockStatisInfo(&pCtx[i], pBlock, &pOperator->pExpr[i].base.colInfo); + + if (pCtx[i].functionId == TSDB_FUNC_ARITHM) { + setArithParams((SArithmeticSupport*)pCtx[i].param[1].pz, &pOperator->pExpr[i], pBlock); } else { - dataBlock = NULL; + SColIndex* pCol = &pOperator->pExpr[i].base.colInfo; + if (TSDB_COL_IS_NORMAL_COL(pCol->flag) || pCol->colId == TSDB_BLOCK_DIST_COLUMN_INDEX) { + SColIndex* pColIndex = &pOperator->pExpr[i].base.colInfo; + SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, pColIndex->colIndex); + + // in case of the block distribution query, the inputBytes is not a constant value. + pCtx[i].pInput = p->pData; + assert(p->info.colId == pColIndex->colId && pCtx[i].inputType == p->info.type);// && pCtx[i].inputBytes == p->info.bytes); + + uint32_t status = aAggs[pCtx[i].functionId].status; + if ((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) { + SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0); + pCtx[i].ptsList = (int64_t*) tsInfo->pData; + } + } + } + } +} + +static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SQLFunctionCtx* pCtx, SSDataBlock* pSDataBlock) { + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + + for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { + int32_t functionId = pCtx[k].functionId; + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { + pCtx[k].startTs = startTs;// this can be set during create the struct + aAggs[functionId].xFunction(&pCtx[k]); } } +} + +static void arithmeticApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t numOfOutput) { + SQuery *pQuery = pRuntimeEnv->pQuery; - return dataBlock; + for (int32_t k = 0; k < numOfOutput; ++k) { + pCtx[k].startTs = pQuery->window.skey; + aAggs[pCtx[k].functionId].xFunction(&pCtx[k]); + } } -static void setNotInterpoWindowKey(SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) { - if (type == RESULT_ROW_START_INTERP) { - for (int32_t k = 0; k < numOfOutput; ++k) { +void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SArray* pDataBlock, TSKEY prevTs, + int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type) { + SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; + SExprInfo* pExpr = pOperator->pExpr; + + SQLFunctionCtx* pCtx = pInfo->pCtx; + + for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { + int32_t functionId = pCtx[k].functionId; + if (functionId != TSDB_FUNC_TWA && functionId != TSDB_FUNC_INTERP) { pCtx[k].start.key = INT64_MIN; + continue; } - } else { - for (int32_t k = 0; k < numOfOutput; ++k) { - pCtx[k].end.key = INT64_MIN; + + SColIndex * pColIndex = &pExpr[k].base.colInfo; + int16_t index = pColIndex->colIndex; + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, index); + + assert(pColInfo->info.colId == pColIndex->colId && curTs != windowKey); + double v1 = 0, v2 = 0, v = 0; + + if (prevRowIndex == -1) { + GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pRuntimeEnv->prevRow[index]); + } else { + GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pColInfo->pData + prevRowIndex * pColInfo->info.bytes); + } + + GET_TYPED_DATA(v2, double, pColInfo->info.type, (char *)pColInfo->pData + curRowIndex * pColInfo->info.bytes); + + if (functionId == TSDB_FUNC_INTERP) { + if (type == RESULT_ROW_START_INTERP) { + pCtx[k].start.key = prevTs; + pCtx[k].start.val = v1; + + pCtx[k].end.key = curTs; + pCtx[k].end.val = v2; + } + } else if (functionId == TSDB_FUNC_TWA) { + SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; + SPoint point2 = (SPoint){.key = curTs, .val = &v2}; + SPoint point = (SPoint){.key = windowKey, .val = &v }; + + taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); + + if (type == RESULT_ROW_START_INTERP) { + pCtx[k].start.key = point.key; + pCtx[k].start.val = v; + } else { + pCtx[k].end.key = point.key; + pCtx[k].end.val = v; + } } } } -// window start key interpolation -static bool setTimeWindowInterpolationStartTs(SQueryRuntimeEnv* pRuntimeEnv, int32_t pos, int32_t numOfRows, - SArray* pDataBlock, TSKEY* tsCols, STimeWindow* win, int16_t type) { +static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SQLFunctionCtx* pCtx, int32_t pos, + int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, STimeWindow* win) { + SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; SQuery* pQuery = pRuntimeEnv->pQuery; + bool ascQuery = QUERY_IS_ASC_QUERY(pQuery); + TSKEY curTs = tsCols[pos]; TSKEY lastTs = *(TSKEY *) pRuntimeEnv->prevRow[0]; // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed. // start exactly from this point, no need to do interpolation - TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win->skey:win->ekey; + TSKEY key = ascQuery? win->skey:win->ekey; if (key == curTs) { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); + setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); return true; } - if (lastTs == INT64_MIN && ((pos == 0 && QUERY_IS_ASC_QUERY(pQuery)) || (pos == (numOfRows - 1) && !QUERY_IS_ASC_QUERY(pQuery)))) { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); + if (lastTs == INT64_MIN && ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery))) { + setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); return true; } int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - TSKEY prevTs = ((pos == 0 && QUERY_IS_ASC_QUERY(pQuery)) || (pos == (numOfRows - 1) && !QUERY_IS_ASC_QUERY(pQuery)))? - lastTs:tsCols[pos - step]; + TSKEY prevTs = ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery))? lastTs:tsCols[pos - step]; - doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, pos - step, curTs, pos, key, RESULT_ROW_START_INTERP); + doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, prevTs, pos - step, curTs, pos, + key, RESULT_ROW_START_INTERP); return true; } -static bool setTimeWindowInterpolationEndTs(SQueryRuntimeEnv* pRuntimeEnv, int32_t endRowIndex, SArray* pDataBlock, TSKEY* tsCols, TSKEY blockEkey, STimeWindow* win) { +static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SQLFunctionCtx* pCtx, + int32_t endRowIndex, SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey, STimeWindow* win) { + SQueryRuntimeEnv *pRuntimeEnv = pOperatorInfo->pRuntimeEnv; SQuery* pQuery = pRuntimeEnv->pQuery; + int32_t numOfOutput = pOperatorInfo->numOfOutput; + TSKEY actualEndKey = tsCols[endRowIndex]; TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win->ekey:win->skey; // not ended in current data block, do not invoke interpolation if ((key > blockEkey && QUERY_IS_ASC_QUERY(pQuery)) || (key < blockEkey && !QUERY_IS_ASC_QUERY(pQuery))) { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); return false; } // there is actual end point of current time window, no interpolation need if (key == actualEndKey) { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); return true; } @@ -1072,61 +1142,35 @@ static bool setTimeWindowInterpolationEndTs(SQueryRuntimeEnv* pRuntimeEnv, int32 assert(nextRowIndex >= 0); TSKEY nextKey = tsCols[nextRowIndex]; - doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, actualEndKey, endRowIndex, nextKey, nextRowIndex, key, RESULT_ROW_END_INTERP); + doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, actualEndKey, endRowIndex, nextKey, + nextRowIndex, key, RESULT_ROW_END_INTERP); return true; } -static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray* pDataBlock, - int32_t rowIndex) { - if (pDataBlock == NULL) { +static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBlock* pBlock, SQLFunctionCtx* pCtx, + SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep) { + SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + SQuery* pQuery = pRuntimeEnv->pQuery; + if (!pQuery->timeWindowInterpo) { return; } - SQuery* pQuery = pRuntimeEnv->pQuery; - for (int32_t k = 0; k < pQuery->numOfCols; ++k) { - SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k); - memcpy(pRuntimeEnv->prevRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes); - } -} + assert(pBlock != NULL); + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); -static TSKEY getStartTsKey(SQuery* pQuery, SDataBlockInfo* pDataBlockInfo, TSKEY* tsCols, int32_t step) { - TSKEY ts = TSKEY_INITIAL_VAL; - - if (tsCols == NULL) { - ts = QUERY_IS_ASC_QUERY(pQuery) ? pDataBlockInfo->window.skey : pDataBlockInfo->window.ekey; - } else { - int32_t offset = GET_COL_DATA_POS(pQuery, 0, step); - ts = tsCols[offset]; - } - - return ts; -} - -static void doWindowBorderInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray *pDataBlock, - SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep) { - if (!pRuntimeEnv->timeWindowInterpo) { - return; - } - - assert(pDataBlock != NULL); - - SQuery* pQuery = pRuntimeEnv->pQuery; - int32_t fillType = pQuery->fillType; - - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - - SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, 0); + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, 0); TSKEY *tsCols = (TSKEY *)(pColInfo->pData); bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); - if (!done) { + if (!done) { // it is not interpolated, now start to generated the interpolated value int32_t startRowIndex = startPos; - bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startRowIndex, pDataBlockInfo->rows, pDataBlock, tsCols, win, fillType); + bool interp = setTimeWindowInterpolationStartTs(pOperatorInfo, pCtx, startRowIndex, pBlock->info.rows, pBlock->pDataBlock, + tsCols, win); if (interp) { setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); } } else { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); + setNotInterpoWindowKey(pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); } // point interpolation does not require the end key time window interpolation. @@ -1139,157 +1183,231 @@ static void doWindowBorderInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SDataBloc if (!done) { int32_t endRowIndex = startPos + (forwardStep - 1) * step; - TSKEY endKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey; - bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, endRowIndex, pDataBlock, tsCols, endKey, win); + TSKEY endKey = QUERY_IS_ASC_QUERY(pQuery)? pBlock->info.window.ekey:pBlock->info.window.skey; + bool interp = setTimeWindowInterpolationEndTs(pOperatorInfo, pCtx, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win); if (interp) { setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); } } else { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP); } } -/** - * todo set the last value for pQueryTableInfo as in rowwiseapplyfunctions - * @param pRuntimeEnv - * @param forwardStep - * @param tsCols - * @param pFields - * @param isDiskFileBlock - * @return the incremental number of output value, so it maybe 0 for fixed number of query, - * such as count/min/max etc. - */ -static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, - SResultRowInfo *pWindowResInfo, __block_search_fn_t searchFn, SArray *pDataBlock) { - SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); +static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, int32_t groupId) { + STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*) pOperatorInfo->info; - SQuery *pQuery = pRuntimeEnv->pQuery; - int64_t groupId = pQuery->current->groupIndex; + SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + int32_t numOfOutput = pOperatorInfo->numOfOutput; + SQuery* pQuery = pRuntimeEnv->pQuery; + + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + bool ascQuery = QUERY_IS_ASC_QUERY(pQuery); + + int32_t prevIndex = curTimeWindowIndex(pResultRowInfo); - TSKEY *tsCols = NULL; - if (pDataBlock != NULL) { - SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, 0); - tsCols = (TSKEY *)(pColInfo->pData); + TSKEY* tsCols = NULL; + if (pSDataBlock->pDataBlock != NULL) { + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, 0); + tsCols = (int64_t*) pColDataInfo->pData; + assert(tsCols[0] == pSDataBlock->info.window.skey && + tsCols[pSDataBlock->info.rows - 1] == pSDataBlock->info.window.ekey); } - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - char *dataBlock = getDataBlock(pRuntimeEnv, &pRuntimeEnv->sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &pQuery->pExpr1[k]); + int32_t startPos = ascQuery? 0 : (pSDataBlock->info.rows - 1); + TSKEY ts = getStartTsKey(pQuery, &pSDataBlock->info.window, tsCols, pSDataBlock->info.rows); + + STimeWindow win = getActiveTimeWindow(pResultRowInfo, ts, pQuery); + bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + + SResultRow* pResult = NULL; + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pResultRowInfo, &win, masterScan, &pResult, groupId, pInfo->pCtx, + numOfOutput, pInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - int32_t prevIndex = curTimeWindowIndex(pWindowResInfo); + int32_t forwardStep = 0; + TSKEY ekey = reviseWindowEkey(pQuery, &win); + forwardStep = + getNumOfRowsInTimeWindow(pQuery, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true); + + // prev time window not interpolation yet. + int32_t curIndex = curTimeWindowIndex(pResultRowInfo); + if (prevIndex != -1 && prevIndex < curIndex && pQuery->timeWindowInterpo) { + for (int32_t j = prevIndex; j < curIndex; ++j) { // previous time window may be all closed already. + SResultRow* pRes = pResultRowInfo->pResult[j]; + if (pRes->closed) { + assert(resultRowInterpolated(pRes, RESULT_ROW_START_INTERP) && + resultRowInterpolated(pRes, RESULT_ROW_END_INTERP)); + continue; + } - TSKEY ts = getStartTsKey(pQuery, pDataBlockInfo, tsCols, step); - STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); + STimeWindow w = pRes->win; + ret = setWindowOutputBufByKey(pRuntimeEnv, pResultRowInfo, &w, masterScan, &pResult, groupId, pInfo->pCtx, + numOfOutput, pInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } - SResultRow* pResult = NULL; - int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId); - if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { - goto _end; - } + assert(!resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); - int32_t forwardStep = 0; - int32_t startPos = pQuery->pos; + doTimeWindowInterpolation(pOperatorInfo, pInfo, pSDataBlock->pDataBlock, *(TSKEY *)pRuntimeEnv->prevRow[0], + -1, tsCols[startPos], startPos, w.ekey, RESULT_ROW_END_INTERP); - TSKEY ekey = reviseWindowEkey(pQuery, &win); - forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, pQuery->pos, ekey, searchFn, true); + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pInfo->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); - // prev time window not interpolation yet. - int32_t curIndex = curTimeWindowIndex(pWindowResInfo); - if (prevIndex != -1 && prevIndex < curIndex && pRuntimeEnv->timeWindowInterpo) { - for(int32_t j = prevIndex; j < curIndex; ++j) { // previous time window may be all closed already. - SResultRow *pRes = pWindowResInfo->pResult[j]; - if (pRes->closed) { - assert(resultRowInterpolated(pRes, RESULT_ROW_START_INTERP) && resultRowInterpolated(pRes, RESULT_ROW_END_INTERP)); - continue; - } + doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &w, startPos, 0, tsCols, pSDataBlock->info.rows, numOfOutput); + } - STimeWindow w = pRes->win; - ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &w, masterScan, &pResult, groupId); - assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); + // restore current time window + ret = setWindowOutputBufByKey(pRuntimeEnv, pResultRowInfo, &win, masterScan, &pResult, groupId, pInfo->pCtx, + numOfOutput, pInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } - int32_t p = QUERY_IS_ASC_QUERY(pQuery)? 0:pDataBlockInfo->rows-1; - doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, *(TSKEY*) pRuntimeEnv->prevRow[0], -1, tsCols[0], p, w.ekey, RESULT_ROW_END_INTERP); - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); + // window start key interpolation + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &win, startPos, forwardStep); + doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &win, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); - doBlockwiseApplyFunctions(pRuntimeEnv, &w, startPos, 0, tsCols, pDataBlockInfo->rows, pDataBlock); - } + STimeWindow nextWin = win; + while (1) { + int32_t prevEndPos = (forwardStep - 1) * step + startPos; + startPos = getNextQualifiedWindow(pQuery, &nextWin, &pSDataBlock->info, tsCols, binarySearchForKey, prevEndPos); + if (startPos < 0) { + break; + } - // restore current time window - ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId); - assert (ret == TSDB_CODE_SUCCESS); + // null data, failed to allocate more memory buffer + int32_t code = setWindowOutputBufByKey(pRuntimeEnv, pResultRowInfo, &nextWin, masterScan, &pResult, groupId, + pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - // window start key interpolation - doWindowBorderInterpolation(pRuntimeEnv, pDataBlockInfo, pDataBlock, pResult, &win, pQuery->pos, forwardStep); - doBlockwiseApplyFunctions(pRuntimeEnv, &win, startPos, forwardStep, tsCols, pDataBlockInfo->rows, pDataBlock); + ekey = reviseWindowEkey(pQuery, &nextWin); + forwardStep = getNumOfRowsInTimeWindow(pQuery, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true); - STimeWindow nextWin = win; - while (1) { - int32_t prevEndPos = (forwardStep - 1) * step + startPos; - startPos = getNextQualifiedWindow(pRuntimeEnv, &nextWin, pDataBlockInfo, tsCols, searchFn, prevEndPos); - if (startPos < 0) { - break; - } + // window start(end) key interpolation + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &nextWin, startPos, forwardStep); + doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &nextWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); + } - // null data, failed to allocate more memory buffer - int32_t code = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &nextWin, masterScan, &pResult, groupId); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - break; + if (pQuery->timeWindowInterpo) { + int32_t rowIndex = ascQuery? (pSDataBlock->info.rows-1):0; + saveDataBlockLastRow(pRuntimeEnv, &pSDataBlock->info, pSDataBlock->pDataBlock, rowIndex); + } + + updateResultRowInfoActiveIndex(pResultRowInfo, pQuery, pQuery->current->lastKey); +} + +static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STableQueryInfo* item = pRuntimeEnv->pQuery->current; + + SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, pInfo->colIndex); + int16_t bytes = pColInfoData->info.bytes; + int16_t type = pColInfoData->info.type; + + if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { + qError("QInfo:%p group by not supported on double/float columns, abort", pRuntimeEnv->qinfo); + return; + } + + for (int32_t j = 0; j < pSDataBlock->info.rows; ++j) { + char* val = ((char*)pColInfoData->pData) + bytes * j; + if (isNull(val, type)) { + continue; + } + + // Compare with the previous row of this column, and do not set the output buffer again if they are identical. + if (pInfo->prevData == NULL || (memcmp(pInfo->prevData, val, bytes) != 0)) { + if (pInfo->prevData == NULL) { + pInfo->prevData = malloc(bytes); } - ekey = reviseWindowEkey(pQuery, &nextWin); - forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, startPos, ekey, searchFn, true); + memcpy(pInfo->prevData, val, bytes); - // window start(end) key interpolation - doWindowBorderInterpolation(pRuntimeEnv, pDataBlockInfo, pDataBlock, pResult, &nextWin, startPos, forwardStep); - doBlockwiseApplyFunctions(pRuntimeEnv, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows, pDataBlock); + int32_t ret = + setGroupResultOutputBuf(pRuntimeEnv, pInfo, pOperator->numOfOutput, val, type, bytes, item->groupIndex); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); + } } - } else { - /* - * the sqlfunctionCtx parameters should be set done before all functions are invoked, - * since the selectivity + tag_prj query needs all parameters been set done. - * tag_prj function are changed to be TSDB_FUNC_TAG_DUMMY - */ - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - int32_t functionId = pQuery->pExpr1[k].base.functionId; - if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { - pCtx[k].startTs = pQuery->window.skey; - aAggs[functionId].xFunction(&pCtx[k]); + for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { + pInfo->binfo.pCtx[k].size = 1; + int32_t functionId = pInfo->binfo.pCtx[k].functionId; + if (functionNeedToExecute(pRuntimeEnv, &pInfo->binfo.pCtx[k], functionId)) { + aAggs[functionId].xFunctionF(&pInfo->binfo.pCtx[k], j); } } } +} + +static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSWindowOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STableQueryInfo* item = pRuntimeEnv->pQuery->current; + + // primary timestamp column + SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, 0); + + bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + SOptrBasicInfo* pBInfo = &pInfo->binfo; + + int64_t gap = pOperator->pRuntimeEnv->pQuery->sw.gap; + pInfo->numOfRows = 0; + + TSKEY* tsList = (TSKEY*)pColInfoData->pData; + for (int32_t j = 0; j < pSDataBlock->info.rows; ++j) { + if (pInfo->prevTs == INT64_MIN) { + pInfo->curWindow.skey = tsList[j]; + pInfo->curWindow.ekey = tsList[j]; + pInfo->prevTs = tsList[j]; + pInfo->numOfRows = 1; + pInfo->start = j; + } else if (tsList[j] - pInfo->prevTs <= gap) { + pInfo->curWindow.ekey = tsList[j]; + pInfo->prevTs = tsList[j]; + pInfo->numOfRows += 1; + pInfo->start = j; + } else { // start a new session window + SResultRow* pResult = NULL; + + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, &pInfo->curWindow, masterScan, + &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput, + pBInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); + } - _end: - if (pRuntimeEnv->timeWindowInterpo) { - int32_t rowIndex = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->rows-1:0; - saveDataBlockLastRow(pRuntimeEnv, pDataBlockInfo, pDataBlock, rowIndex); + doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, + pSDataBlock->info.rows, pOperator->numOfOutput); + + pInfo->curWindow.skey = tsList[j]; + pInfo->curWindow.ekey = tsList[j]; + pInfo->prevTs = tsList[j]; + pInfo->numOfRows = 1; + pInfo->start = j; + } } -} -static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) { - SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; + SResultRow* pResult = NULL; - // not assign result buffer yet, add new result buffer, TODO remove it - char* d = pData; - int16_t len = bytes; - if (type == TSDB_DATA_TYPE_BINARY||type == TSDB_DATA_TYPE_NCHAR) { - d = varDataVal(pData); - len = varDataLen(pData); - } else if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { - SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); - qError("QInfo:%p group by not supported on double/float columns, abort", pQInfo); - return -1; + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, &pInfo->curWindow, masterScan, + &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput, + pBInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } - SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->resultRowInfo, d, len, true, groupIndex); - assert (pResultRow != NULL); + doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, + pSDataBlock->info.rows, pOperator->numOfOutput); +} +static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { int64_t v = -1; GET_TYPED_DATA(v, int64_t, type, pData); if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { @@ -1303,94 +1421,58 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat pResultRow->win.skey = v; pResultRow->win.ekey = v; } +} + +static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SGroupbyOperatorInfo *pInfo, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) { + SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; + + int32_t *rowCellInfoOffset = pInfo->binfo.rowCellInfoOffset; + SResultRowInfo *pResultRowInfo = &pInfo->binfo.resultRowInfo; + SQLFunctionCtx *pCtx = pInfo->binfo.pCtx; + + // not assign result buffer yet, add new result buffer, TODO remove it + char* d = pData; + int16_t len = bytes; + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + d = varDataVal(pData); + len = varDataLen(pData); + } + SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pResultRowInfo, d, len, true, groupIndex); + assert (pResultRow != NULL); + + setResultRowKey(pResultRow, pData, type); if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, groupIndex, pRuntimeEnv->numOfRowsPerPage); + int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, groupIndex, pRuntimeEnv->pQuery->resultRowSize); if (ret != 0) { return -1; } } - setResultOutputBuf(pRuntimeEnv, pResultRow); - initCtxOutputBuf(pRuntimeEnv); + setResultOutputBuf(pRuntimeEnv, pResultRow, pCtx, numOfCols, rowCellInfoOffset); + initCtxOutputBuffer(pCtx, numOfCols); return TSDB_CODE_SUCCESS; } -static char *getGroupbyColumnData(SQuery *pQuery, int16_t *type, int16_t *bytes, SArray* pDataBlock) { - SSqlGroupbyExpr *pGroupbyExpr = pQuery->pGroupbyExpr; - +static int32_t getGroupbyColumnIndex(SSqlGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock) { for (int32_t k = 0; k < pGroupbyExpr->numOfGroupCols; ++k) { SColIndex* pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, k); if (TSDB_COL_IS_TAG(pColIndex->flag)) { continue; } - int16_t colIndex = -1; int32_t colId = pColIndex->colId; - for (int32_t i = 0; i < pQuery->numOfCols; ++i) { - if (pQuery->colList[i].colId == colId) { - colIndex = i; - break; - } - } - - assert(colIndex >= 0 && colIndex < pQuery->numOfCols); - - *type = pQuery->colList[colIndex].type; - *bytes = pQuery->colList[colIndex].bytes; - /* - * the colIndex is acquired from the first tables of all qualified tables in this vnode during query prepare - * stage, the remain tables may not have the required column in cache actually. So, the validation of required - * column in cache with the corresponding schema is reinforced. - */ - int32_t numOfCols = (int32_t)taosArrayGetSize(pDataBlock); - - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData *p = taosArrayGet(pDataBlock, i); - if (pColIndex->colId == p->info.colId) { - return p->pData; + for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); + if (pColInfo->info.colId == colId) { + return i; } } } - return NULL; -} - -static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - STSElem elem = tsBufGetElem(pRuntimeEnv->pTsBuf); - SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - - // compare tag first - if (tVariantCompare(&pCtx[0].tag, elem.tag) != 0) { - return TS_JOIN_TAG_NOT_EQUALS; - } - - TSKEY key = *(TSKEY *)((char*)pCtx[0].pInput + TSDB_KEYSIZE * offset); - -#if defined(_DEBUG_VIEW) - printf("elem in comp ts file:%" PRId64 ", key:%" PRId64 ", tag:%"PRIu64", query order:%d, ts order:%d, traverse:%d, index:%d\n", - elem.ts, key, elem.tag.i64, pQuery->order.order, pRuntimeEnv->pTsBuf->tsOrder, - pRuntimeEnv->pTsBuf->cur.order, pRuntimeEnv->pTsBuf->cur.tsIndex); -#endif - - if (QUERY_IS_ASC_QUERY(pQuery)) { - if (key < elem.ts) { - return TS_JOIN_TS_NOT_EQUALS; - } else if (key > elem.ts) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_INCONSISTAN); - } - } else { - if (key > elem.ts) { - return TS_JOIN_TS_NOT_EQUALS; - } else if (key < elem.ts) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_INCONSISTAN); - } - } - - return TS_JOIN_TS_EQUAL; + assert(0); + return -1; } static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId) { @@ -1415,613 +1497,313 @@ static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx return pCtx->param[0].i64 == pQuery->order.order; } - // in the supplementary scan, only the following functions need to be executed - if (IS_REVERSE_SCAN(pRuntimeEnv)) { + // in the reverse table scan, only the following functions need to be executed + if (IS_REVERSE_SCAN(pRuntimeEnv) || + (pRuntimeEnv->scanFlag == REPEAT_SCAN && functionId != TSDB_FUNC_STDDEV && functionId != TSDB_FUNC_PERCT)) { return false; } return true; } -void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv *pRuntimeEnv, SArray *pDataBlock, TSKEY prevTs, - int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, - int32_t type) { - SQuery *pQuery = pRuntimeEnv->pQuery; - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - int32_t functionId = pQuery->pExpr1[k].base.functionId; - if (functionId != TSDB_FUNC_TWA && functionId != TSDB_FUNC_INTERP) { - pRuntimeEnv->pCtx[k].start.key = INT64_MIN; - continue; - } - - SColIndex * pColIndex = &pQuery->pExpr1[k].base.colInfo; - int16_t index = pColIndex->colIndex; - SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, index); +void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColIndex* pColIndex) { + SDataStatis *pStatis = NULL; - assert(pColInfo->info.colId == pColIndex->colId && curTs != windowKey); - double v1 = 0, v2 = 0, v = 0; + if (pSDataBlock->pBlockStatis != NULL && TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { + pStatis = &pSDataBlock->pBlockStatis[pColIndex->colIndex]; - if (prevRowIndex == -1) { - GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pRuntimeEnv->prevRow[index]); - } else { - GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pColInfo->pData + prevRowIndex * pColInfo->info.bytes); - } + pCtx->preAggVals.statis = *pStatis; + pCtx->preAggVals.isSet = true; + assert(pCtx->preAggVals.statis.numOfNull <= pSDataBlock->info.rows); + } else { + pCtx->preAggVals.isSet = false; + } - GET_TYPED_DATA(v2, double, pColInfo->info.type, (char *)pColInfo->pData + curRowIndex * pColInfo->info.bytes); + pCtx->hasNull = hasNullRv(pColIndex, pStatis); - SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; - SPoint point2 = (SPoint){.key = curTs, .val = &v2}; - SPoint point = (SPoint){.key = windowKey, .val = &v}; + // set the statistics data for primary time stamp column + if (pCtx->functionId == TSDB_FUNC_SPREAD && pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + pCtx->preAggVals.isSet = true; + pCtx->preAggVals.statis.min = pSDataBlock->info.window.skey; + pCtx->preAggVals.statis.max = pSDataBlock->info.window.ekey; + } +} - if (functionId == TSDB_FUNC_TWA) { - taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); +// set the output buffer for the selectivity + tag query +static int32_t setCtxTagColumnInfo(SQLFunctionCtx *pCtx, int32_t numOfOutput) { + if (!isSelectivityWithTagsQuery(pCtx, numOfOutput)) { + return TSDB_CODE_SUCCESS; + } - if (type == RESULT_ROW_START_INTERP) { - pRuntimeEnv->pCtx[k].start.key = point.key; - pRuntimeEnv->pCtx[k].start.val = v; - } else { - pRuntimeEnv->pCtx[k].end.key = point.key; - pRuntimeEnv->pCtx[k].end.val = v; - } - } else { - if (type == RESULT_ROW_START_INTERP) { - pRuntimeEnv->pCtx[k].start.key = prevTs; - pRuntimeEnv->pCtx[k].start.val = v1; + int32_t num = 0; + int16_t tagLen = 0; - pRuntimeEnv->pCtx[k].end.key = curTs; - pRuntimeEnv->pCtx[k].end.val = v2; - } - } + SQLFunctionCtx* p = NULL; + SQLFunctionCtx** pTagCtx = calloc(numOfOutput, POINTER_BYTES); + if (pTagCtx == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; } -} -static void setTimeWindowSKeyInterp(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY ts, int32_t offset, SResultRow* pResult, STimeWindow* win) { - SQuery* pQuery = pRuntimeEnv->pQuery; - - bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); - if (!done) { - TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win->skey:win->ekey; - if (key == ts) { - setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - } else if (prevTs != INT64_MIN && ((QUERY_IS_ASC_QUERY(pQuery) && prevTs < key) || (!QUERY_IS_ASC_QUERY(pQuery) && prevTs > key))) { - doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_START_INTERP); - setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + for (int32_t i = 0; i < numOfOutput; ++i) { + int32_t functionId = pCtx[i].functionId; + + if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { + tagLen += pCtx[i].outputBytes; + pTagCtx[num++] = &pCtx[i]; + } else if ((aAggs[functionId].status & TSDB_FUNCSTATE_SELECTIVITY) != 0) { + p = &pCtx[i]; + } else if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG) { + // tag function may be the group by tag column + // ts may be the required primary timestamp column + continue; } else { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); - } - - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP); - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - pRuntimeEnv->pCtx[k].size = 1; + // the column may be the normal column, group by normal_column, the functionId is TSDB_FUNC_PRJ } + } + if (p != NULL) { + p->tagInfo.pTagCtxList = pTagCtx; + p->tagInfo.numOfTagCols = num; + p->tagInfo.tagsLen = tagLen; } else { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); + tfree(pTagCtx); } + + return TSDB_CODE_SUCCESS; } -static void setTimeWindowEKeyInterp(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY ts, int32_t offset, SResultRow* pResult, STimeWindow* win) { +static SQLFunctionCtx* createSQLFunctionCtx(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, + int32_t** rowCellInfoOffset) { SQuery* pQuery = pRuntimeEnv->pQuery; - TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win->ekey:win->skey; - doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_END_INTERP); - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - pRuntimeEnv->pCtx[i].size = 0; + SQLFunctionCtx * pFuncCtx = (SQLFunctionCtx *)calloc(numOfOutput, sizeof(SQLFunctionCtx)); + if (pFuncCtx == NULL) { + return NULL; } -} - -static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, - SResultRowInfo *pWindowResInfo, SArray *pDataBlock) { - SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); - - SQuery *pQuery = pRuntimeEnv->pQuery; - STableQueryInfo* item = pQuery->current; - int64_t groupId = item->groupIndex; - - SColumnInfoData* pColumnInfoData = (SColumnInfoData *)taosArrayGet(pDataBlock, 0); - - TSKEY *tsCols = (pColumnInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP)? (TSKEY*) pColumnInfoData->pData:NULL; - bool groupbyColumnValue = pRuntimeEnv->groupbyColumn; - - int16_t type = 0; - int16_t bytes = 0; - - char *groupbyColumnData = NULL; - if (groupbyColumnValue) { - groupbyColumnData = getGroupbyColumnData(pQuery, &type, &bytes, pDataBlock); + *rowCellInfoOffset = calloc(numOfOutput, sizeof(int32_t)); + if (*rowCellInfoOffset == 0) { + tfree(pFuncCtx); + return NULL; } - SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - char *dataBlock = getDataBlock(pRuntimeEnv, &pRuntimeEnv->sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &pQuery->pExpr1[k]); - pCtx[k].size = 1; - } + for (int32_t i = 0; i < numOfOutput; ++i) { + SSqlFuncMsg *pSqlFuncMsg = &pExpr[i].base; + SQLFunctionCtx* pCtx = &pFuncCtx[i]; - // set the input column data - for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { - SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; - pFilterInfo->pData = getDataBlockImpl(pDataBlock, pFilterInfo->info.colId); - assert(pFilterInfo->pData != NULL); - } + SColIndex *pIndex = &pSqlFuncMsg->colInfo; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + if (TSDB_COL_REQ_NULL(pIndex->flag)) { + pCtx->requireNull = true; + pIndex->flag &= ~(TSDB_COL_NULL); + } else { + pCtx->requireNull = false; + } - // from top to bottom in desc - // from bottom to top in asc order - if (pRuntimeEnv->pTsBuf != NULL) { - qDebug("QInfo:%p process data rows, numOfRows:%d, query order:%d, ts comp order:%d", pQInfo, pDataBlockInfo->rows, - pQuery->order.order, pRuntimeEnv->pTsBuf->cur.order); - } + pCtx->inputBytes = pSqlFuncMsg->colBytes; + pCtx->inputType = pSqlFuncMsg->colType; - int32_t offset = -1; - TSKEY prevTs = *(TSKEY*) pRuntimeEnv->prevRow[0]; - int32_t prevRowIndex = -1; + pCtx->ptsOutputBuf = NULL; - for (int32_t j = 0; j < pDataBlockInfo->rows; ++j) { - offset = GET_COL_DATA_POS(pQuery, j, step); + pCtx->outputBytes = pExpr[i].bytes; + pCtx->outputType = pExpr[i].type; - if (pRuntimeEnv->pTsBuf != NULL) { - int32_t ret = doTSJoinFilter(pRuntimeEnv, offset); - if (ret == TS_JOIN_TAG_NOT_EQUALS) { - break; - } else if (ret == TS_JOIN_TS_NOT_EQUALS) { + pCtx->order = pQuery->order.order; + pCtx->functionId = pSqlFuncMsg->functionId; + pCtx->stableQuery = pQuery->stableQuery; + pCtx->interBufBytes = pExpr[i].interBytes; + pCtx->start.key = INT64_MIN; + pCtx->end.key = INT64_MIN; + + pCtx->numOfParams = pSqlFuncMsg->numOfParams; + for (int32_t j = 0; j < pCtx->numOfParams; ++j) { + int16_t type = pSqlFuncMsg->arg[j].argType; + int16_t bytes = pSqlFuncMsg->arg[j].argBytes; + if (pSqlFuncMsg->functionId == TSDB_FUNC_STDDEV_DST) { continue; + } + + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + tVariantCreateFromBinary(&pCtx->param[j], pSqlFuncMsg->arg[j].argValue.pz, bytes, type); } else { - assert(ret == TS_JOIN_TS_EQUAL); + tVariantCreateFromBinary(&pCtx->param[j], (char *)&pSqlFuncMsg->arg[j].argValue.i64, bytes, type); } } - if (pQuery->numOfFilterCols > 0 && (!doFilterData(pQuery, offset))) { - continue; - } + // set the order information for top/bottom query + int32_t functionId = pCtx->functionId; - // interval window query, decide the time window according to the primary timestamp - if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - int32_t prevWindowIndex = curTimeWindowIndex(pWindowResInfo); - int64_t ts = tsCols[offset]; + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { + int32_t f = pExpr[0].base.functionId; + assert(f == TSDB_FUNC_TS || f == TSDB_FUNC_TS_DUMMY); - STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); - - SResultRow* pResult = NULL; - int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId); - if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { // null data, too many state code - goto _end; - } - - // window start key interpolation - if (pRuntimeEnv->timeWindowInterpo) { - // check for the time window end time interpolation - int32_t curIndex = curTimeWindowIndex(pWindowResInfo); - if (prevWindowIndex != -1 && prevWindowIndex < curIndex) { - for (int32_t k = prevWindowIndex; k < curIndex; ++k) { - SResultRow *pRes = pWindowResInfo->pResult[k]; - if (pRes->closed) { - assert(resultRowInterpolated(pResult, RESULT_ROW_START_INTERP) && resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); - continue; - } - - ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &pRes->win, masterScan, &pResult, groupId); - assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); - - setTimeWindowEKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &pRes->win); - doRowwiseApplyFunctions(pRuntimeEnv, &pRes->win, offset); - } - - // restore current time window - ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - continue; - } - } - - setTimeWindowSKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &win); - } - - doRowwiseApplyFunctions(pRuntimeEnv, &win, offset); - int32_t index = pWindowResInfo->curIndex; - - STimeWindow nextWin = win; - while (1) { - getNextTimeWindow(pQuery, &nextWin); - if ((nextWin.skey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (nextWin.ekey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) { - break; - } - - if (ts < nextWin.skey || ts > nextWin.ekey) { - break; - } - - // null data, failed to allocate more memory buffer - int32_t code = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &nextWin, masterScan, &pResult, groupId); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - break; - } - - setTimeWindowSKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &nextWin); - doRowwiseApplyFunctions(pRuntimeEnv, &nextWin, offset); - } - - // restore the index, add the result row will move the index - pWindowResInfo->curIndex = index; - } else { // other queries - // decide which group this rows belongs to according to current state value - char* val = NULL; - if (groupbyColumnValue) { - val = groupbyColumnData + bytes * offset; - if (isNull(val, type)) { // ignore the null value - continue; - } - - int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, val, type, bytes, item->groupIndex); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); - } - } - - if (pRuntimeEnv->stabledev) { - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - int32_t functionId = pQuery->pExpr1[k].base.functionId; - if (functionId != TSDB_FUNC_STDDEV_DST) { - continue; - } + pCtx->param[2].i64 = pQuery->order.order; + pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; + pCtx->param[3].i64 = functionId; + pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT; - pRuntimeEnv->pCtx[k].param[0].arr = NULL; - pRuntimeEnv->pCtx[k].param[0].nType = TSDB_DATA_TYPE_INT; // avoid freeing the memory by setting the type to be int - - // todo opt perf - int32_t numOfGroup = (int32_t)taosArrayGetSize(pRuntimeEnv->prevResult); - for (int32_t i = 0; i < numOfGroup; ++i) { - SInterResult *p = taosArrayGet(pRuntimeEnv->prevResult, i); - if (memcmp(p->tags, val, bytes) == 0) { - int32_t numOfCols = (int32_t)taosArrayGetSize(p->pResult); - for (int32_t f = 0; f < numOfCols; ++f) { - SStddevInterResult *pres = taosArrayGet(p->pResult, f); - if (pres->colId == pQuery->pExpr1[k].base.colInfo.colId) { - pRuntimeEnv->pCtx[k].param[0].arr = pres->pResult; - break; - } - } - } + pCtx->param[1].i64 = pQuery->order.orderColId; + } else if (functionId == TSDB_FUNC_INTERP) { + pCtx->param[2].i64 = (int8_t)pQuery->fillType; + if (pQuery->fillVal != NULL) { + if (isNull((const char *)&pQuery->fillVal[i], pCtx->inputType)) { + pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; + } else { // todo refactor, tVariantCreateFromBinary should handle the NULL value + if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) { + tVariantCreateFromBinary(&pCtx->param[1], (char *)&pQuery->fillVal[i], pCtx->inputBytes, pCtx->inputType); } } } - - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - int32_t functionId = pQuery->pExpr1[k].base.functionId; - if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { - aAggs[functionId].xFunctionF(&pCtx[k], offset); - } - } - } - - prevTs = tsCols[offset]; - prevRowIndex = offset; - - if (pRuntimeEnv->pTsBuf != NULL) { - // if timestamp filter list is empty, quit current query - if (!tsBufNextPos(pRuntimeEnv->pTsBuf)) { - setQueryStatus(pQuery, QUERY_COMPLETED); - break; - } + } else if (functionId == TSDB_FUNC_TS_COMP) { + pCtx->param[0].i64 = pQuery->vgId; //TODO this should be the parameter from client + pCtx->param[0].nType = TSDB_DATA_TYPE_BIGINT; + } else if (functionId == TSDB_FUNC_TWA) { + pCtx->param[1].i64 = pQuery->window.skey; + pCtx->param[1].nType = TSDB_DATA_TYPE_BIGINT; + pCtx->param[2].i64 = pQuery->window.ekey; + pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; + } else if (functionId == TSDB_FUNC_ARITHM) { + pCtx->param[1].pz = (char*) &pRuntimeEnv->sasArray[i]; } } - _end: - assert(offset >= 0 && tsCols != NULL); - if (prevTs != INT64_MIN && prevTs != *(int64_t*)pRuntimeEnv->prevRow[0]) { - assert(prevRowIndex >= 0); - item->lastKey = prevTs + step; - } - - // In case of all rows in current block are not qualified - if (pRuntimeEnv->timeWindowInterpo && prevRowIndex != -1) { - saveDataBlockLastRow(pRuntimeEnv, pDataBlockInfo, pDataBlock, prevRowIndex); - } - - if (pRuntimeEnv->pTsBuf != NULL) { - item->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); - } -} - -static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, - SDataStatis *pStatis, __block_search_fn_t searchFn, SArray *pDataBlock) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - STableQueryInfo* pTableQueryInfo = pQuery->current; - SResultRowInfo* pResultRowInfo = &pRuntimeEnv->resultRowInfo; - - if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->groupbyColumn) { - rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, pDataBlock); - } else { - blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, searchFn, pDataBlock); + for(int32_t i = 1; i < numOfOutput; ++i) { + (*rowCellInfoOffset)[i] = (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowCellInfo) + + pExpr[i - 1].interBytes * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pQuery->topBotQuery, pQuery->stableQuery)); } - // update the lastkey of current table for projection/aggregation query - TSKEY lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pDataBlockInfo->window.ekey : pDataBlockInfo->window.skey; - pTableQueryInfo->lastKey = lastKey + GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - - // interval query with limit applied - int32_t numOfRes = 0; - if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyColumn) { - numOfRes = pResultRowInfo->size; - updateResultRowIndex(pResultRowInfo, pTableQueryInfo, QUERY_IS_ASC_QUERY(pQuery), pRuntimeEnv->timeWindowInterpo); - } else { // projection query - numOfRes = (int32_t) getNumOfResult(pRuntimeEnv); - - // update the number of output result - if (numOfRes > 0 && pQuery->checkResultBuf == 1) { - assert(numOfRes >= pQuery->rec.rows); - pQuery->rec.rows = numOfRes; - - if (numOfRes >= pQuery->rec.threshold) { - setQueryStatus(pQuery, QUERY_RESBUF_FULL); - } - - if ((pQuery->limit.limit >= 0) && (pQuery->limit.limit + pQuery->limit.offset) <= numOfRes) { - setQueryStatus(pQuery, QUERY_COMPLETED); - } - - if (((pTableQueryInfo->lastKey > pTableQueryInfo->win.ekey) && QUERY_IS_ASC_QUERY(pQuery)) || - ((pTableQueryInfo->lastKey < pTableQueryInfo->win.ekey) && (!QUERY_IS_ASC_QUERY(pQuery)))) { - setQueryStatus(pQuery, QUERY_COMPLETED); - } - } - } + setCtxTagColumnInfo(pFuncCtx, numOfOutput); - return numOfRes; + return pFuncCtx; } -void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo, - SDataStatis *pStatis, SExprInfo* pExprInfo) { - - int32_t functionId = pExprInfo->base.functionId; - int32_t colId = pExprInfo->base.colInfo.colId; - - SDataStatis *tpField = NULL; - pCtx->hasNull = hasNullValue(&pExprInfo->base.colInfo, pStatis, &tpField); - - if (tpField != NULL) { - pCtx->preAggVals.isSet = true; - pCtx->preAggVals.statis = *tpField; - assert(pCtx->preAggVals.statis.numOfNull <= pBlockInfo->rows); - } else { - pCtx->preAggVals.isSet = false; - } - - pCtx->preAggVals.dataBlockLoaded = (inputData != NULL); - - // limit/offset query will affect this value - pCtx->size = QUERY_IS_ASC_QUERY(pQuery) ? pBlockInfo->rows - pQuery->pos : pQuery->pos + 1; - - // set the start position in current block - int32_t offset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos: (pQuery->pos - pCtx->size + 1); - if (inputData != NULL) { - pCtx->pInput = (char*)inputData + offset * pCtx->inputBytes; - } - - uint32_t status = aAggs[functionId].status; - if (((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) && (tsCol != NULL)) { - pCtx->ptsList = tsCol + offset; - } - - if (functionId == TSDB_FUNC_SPREAD) { // set the statistics data for primary time stamp column - if (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - pCtx->preAggVals.isSet = true; - pCtx->preAggVals.statis.min = pBlockInfo->window.skey; - pCtx->preAggVals.statis.max = pBlockInfo->window.ekey; - } +static void* destroySQLFunctionCtx(SQLFunctionCtx* pCtx, int32_t numOfOutput) { + if (pCtx == NULL) { + return NULL; } -} -// set the output buffer for the selectivity + tag query -static int32_t setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx) { - SQuery* pQuery = pRuntimeEnv->pQuery; - - if (isSelectivityWithTagsQuery(pQuery)) { - int32_t num = 0; - int16_t tagLen = 0; - - SQLFunctionCtx *p = NULL; - SQLFunctionCtx **pTagCtx = calloc(pQuery->numOfOutput, POINTER_BYTES); - if (pTagCtx == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; + for (int32_t i = 0; i < numOfOutput; ++i) { + for (int32_t j = 0; j < pCtx[i].numOfParams; ++j) { + tVariantDestroy(&pCtx[i].param[j]); } - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SSqlFuncMsg *pSqlFuncMsg = &pQuery->pExpr1[i].base; - - if (pSqlFuncMsg->functionId == TSDB_FUNC_TAG_DUMMY || pSqlFuncMsg->functionId == TSDB_FUNC_TS_DUMMY) { - tagLen += pCtx[i].outputBytes; - pTagCtx[num++] = &pCtx[i]; - } else if ((aAggs[pSqlFuncMsg->functionId].status & TSDB_FUNCSTATE_SELECTIVITY) != 0) { - p = &pCtx[i]; - } else if (pSqlFuncMsg->functionId == TSDB_FUNC_TS || pSqlFuncMsg->functionId == TSDB_FUNC_TAG) { - // tag function may be the group by tag column - // ts may be the required primary timestamp column - continue; - } else { - // the column may be the normal column, group by normal_column, the functionId is TSDB_FUNC_PRJ - } - } - if (p != NULL) { - p->tagInfo.pTagCtxList = pTagCtx; - p->tagInfo.numOfTagCols = num; - p->tagInfo.tagsLen = tagLen; - } else { - tfree(pTagCtx); - } + tVariantDestroy(&pCtx[i].tag); + tfree(pCtx[i].tagInfo.pTagCtxList); } - return TSDB_CODE_SUCCESS; + tfree(pCtx); + return NULL; } -static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfTables, int16_t order, int32_t vgId) { - qDebug("QInfo:%p setup runtime env", GET_QINFO_ADDR(pRuntimeEnv)); +static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfTables) { + qDebug("QInfo:%p setup runtime env", pRuntimeEnv->qinfo); SQuery *pQuery = pRuntimeEnv->pQuery; - pRuntimeEnv->interBufSize = getOutputInterResultBufSize(pQuery); - pRuntimeEnv->summary.tableInfoSize += (numOfTables * sizeof(STableQueryInfo)); + pRuntimeEnv->prevGroupId = INT32_MIN; + pRuntimeEnv->pQuery = pQuery; pRuntimeEnv->pResultRowHashTable = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); pRuntimeEnv->keyBuf = malloc(pQuery->maxSrcColumnSize + sizeof(int64_t)); pRuntimeEnv->pool = initResultRowPool(getResultRowSize(pRuntimeEnv)); pRuntimeEnv->prevRow = malloc(POINTER_BYTES * pQuery->numOfCols + pQuery->srcRowSize); pRuntimeEnv->tagVal = malloc(pQuery->tagLen); - pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutput, sizeof(SQLFunctionCtx)); - pRuntimeEnv->offset = calloc(pQuery->numOfOutput, sizeof(int16_t)); - pRuntimeEnv->rowCellInfoOffset = calloc(pQuery->numOfOutput, sizeof(int32_t)); + pRuntimeEnv->currentOffset = pQuery->limit.offset; + + // NOTE: pTableCheckInfo need to update the query time range and the lastKey info + pRuntimeEnv->pTableRetrieveTsMap = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + pRuntimeEnv->sasArray = calloc(pQuery->numOfOutput, sizeof(SArithmeticSupport)); - if (pRuntimeEnv->offset == NULL || pRuntimeEnv->pCtx == NULL || pRuntimeEnv->rowCellInfoOffset == NULL || - pRuntimeEnv->sasArray == NULL || pRuntimeEnv->pResultRowHashTable == NULL || pRuntimeEnv->keyBuf == NULL || - pRuntimeEnv->prevRow == NULL || pRuntimeEnv->tagVal == NULL) { + if (pRuntimeEnv->sasArray == NULL || pRuntimeEnv->pResultRowHashTable == NULL || pRuntimeEnv->keyBuf == NULL || + pRuntimeEnv->prevRow == NULL || pRuntimeEnv->tagVal == NULL) { goto _clean; } - char* start = POINTER_BYTES * pQuery->numOfCols + (char*) pRuntimeEnv->prevRow; - pRuntimeEnv->prevRow[0] = start; - for(int32_t i = 1; i < pQuery->numOfCols; ++i) { - pRuntimeEnv->prevRow[i] = pRuntimeEnv->prevRow[i - 1] + pQuery->colList[i-1].bytes; - } - - pRuntimeEnv->offset[0] = 0; - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SSqlFuncMsg *pSqlFuncMsg = &pQuery->pExpr1[i].base; - - SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; - SColIndex * pIndex = &pSqlFuncMsg->colInfo; - - if (TSDB_COL_REQ_NULL(pIndex->flag)) { - pCtx->requireNull = true; - pIndex->flag &= ~(TSDB_COL_NULL); - } else { - pCtx->requireNull = false; - } - - int32_t index = pSqlFuncMsg->colInfo.colIndex; - if (TSDB_COL_IS_TAG(pIndex->flag)) { - if (pIndex->colId == TSDB_TBNAME_COLUMN_INDEX) { // todo refactor - SSchema *s = tGetTbnameColumnSchema(); - - pCtx->inputBytes = s->bytes; - pCtx->inputType = s->type; - } else if (pIndex->colId == TSDB_BLOCK_DIST_COLUMN_INDEX) { - SSchema s = tGetBlockDistColumnSchema(); - pCtx->inputBytes = s.bytes; - pCtx->inputType = s.type; - } else { - pCtx->inputBytes = pQuery->tagColList[index].bytes; - pCtx->inputType = pQuery->tagColList[index].type; - } - } else if (TSDB_COL_IS_UD_COL(pIndex->flag)) { - pCtx->inputBytes = pSqlFuncMsg->arg[0].argBytes; - pCtx->inputType = pSqlFuncMsg->arg[0].argType; - } else { - pCtx->inputBytes = pQuery->colList[index].bytes; - pCtx->inputType = pQuery->colList[index].type; + if (pQuery->numOfCols) { + char* start = POINTER_BYTES * pQuery->numOfCols + (char*) pRuntimeEnv->prevRow; + pRuntimeEnv->prevRow[0] = start; + for(int32_t i = 1; i < pQuery->numOfCols; ++i) { + pRuntimeEnv->prevRow[i] = pRuntimeEnv->prevRow[i - 1] + pQuery->colList[i-1].bytes; } - assert(isValidDataType(pCtx->inputType)); - pCtx->ptsOutputBuf = NULL; + *(int64_t*) pRuntimeEnv->prevRow[0] = INT64_MIN; + } - pCtx->outputBytes = pQuery->pExpr1[i].bytes; - pCtx->outputType = pQuery->pExpr1[i].type; + qDebug("QInfo:%p init runtime environment completed", pRuntimeEnv->qinfo); - pCtx->order = pQuery->order.order; - pCtx->functionId = pSqlFuncMsg->functionId; - pCtx->stableQuery = pRuntimeEnv->stableQuery; - pCtx->interBufBytes = pQuery->pExpr1[i].interBytes; - pCtx->start.key = INT64_MIN; - pCtx->end.key = INT64_MIN; + // group by normal column, sliding window query, interval query are handled by interval query processor + // interval (down sampling operation) + if (onlyQueryTags(pQuery)) { // do nothing for tags query + + } else if (QUERY_IS_INTERVAL_QUERY(pQuery)) { + if (pQuery->stableQuery) { + pRuntimeEnv->proot = createMultiTableTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->pTableScanner, + pQuery->pExpr1, pQuery->numOfOutput); + setTableScanFilterOperatorInfo(pRuntimeEnv->pTableScanner->info, pRuntimeEnv->proot); + } else { + pRuntimeEnv->proot = + createTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->pTableScanner, pQuery->pExpr1, pQuery->numOfOutput); + setTableScanFilterOperatorInfo(pRuntimeEnv->pTableScanner->info, pRuntimeEnv->proot); - pCtx->numOfParams = pSqlFuncMsg->numOfParams; - for (int32_t j = 0; j < pCtx->numOfParams; ++j) { - int16_t type = pSqlFuncMsg->arg[j].argType; - int16_t bytes = pSqlFuncMsg->arg[j].argBytes; - if (pSqlFuncMsg->functionId == TSDB_FUNC_STDDEV_DST) { - continue; + if (pQuery->pExpr2 != NULL) { + pRuntimeEnv->proot = + createArithOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQuery->pExpr2, pQuery->numOfExpr2); } - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - tVariantCreateFromBinary(&pCtx->param[j], pSqlFuncMsg->arg[j].argValue.pz, bytes, type); - } else { - tVariantCreateFromBinary(&pCtx->param[j], (char *)&pSqlFuncMsg->arg[j].argValue.i64, bytes, type); + if (pQuery->fillType != TSDB_FILL_NONE && !isPointInterpoQuery(pQuery)) { + SOperatorInfo* pInfo = pRuntimeEnv->proot; + pRuntimeEnv->proot = createFillOperatorInfo(pRuntimeEnv, pInfo, pInfo->pExpr, pInfo->numOfOutput); } } - // set the order information for top/bottom query - int32_t functionId = pCtx->functionId; + } else if (pQuery->groupbyColumn) { + pRuntimeEnv->proot = + createGroupbyOperatorInfo(pRuntimeEnv, pRuntimeEnv->pTableScanner, pQuery->pExpr1, pQuery->numOfOutput); + setTableScanFilterOperatorInfo(pRuntimeEnv->pTableScanner->info, pRuntimeEnv->proot); - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - int32_t f = pQuery->pExpr1[0].base.functionId; - assert(f == TSDB_FUNC_TS || f == TSDB_FUNC_TS_DUMMY); + if (pQuery->pExpr2 != NULL) { + pRuntimeEnv->proot = createArithOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQuery->pExpr2, pQuery->numOfExpr2); + } + } else if (pQuery->sw.gap > 0) { + pRuntimeEnv->proot = createSWindowOperatorInfo(pRuntimeEnv, pRuntimeEnv->pTableScanner, pQuery->pExpr1, pQuery->numOfOutput); + setTableScanFilterOperatorInfo(pRuntimeEnv->pTableScanner->info, pRuntimeEnv->proot); - pCtx->param[2].i64 = order; - pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; - pCtx->param[3].i64 = functionId; - pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT; + if (pQuery->pExpr2 != NULL) { + pRuntimeEnv->proot = createArithOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQuery->pExpr2, pQuery->numOfExpr2); + } + } else if (isFixedOutputQuery(pQuery)) { + if (pQuery->stableQuery && !isTsCompQuery(pQuery)) { + pRuntimeEnv->proot = + createMultiTableAggOperatorInfo(pRuntimeEnv, pRuntimeEnv->pTableScanner, pQuery->pExpr1, pQuery->numOfOutput); + } else { + pRuntimeEnv->proot = + createAggregateOperatorInfo(pRuntimeEnv, pRuntimeEnv->pTableScanner, pQuery->pExpr1, pQuery->numOfOutput); + } - pCtx->param[1].i64 = pQuery->order.orderColId; - } else if (functionId == TSDB_FUNC_INTERP) { - pCtx->param[2].i64 = (int8_t)pQuery->fillType; - if (pQuery->fillVal != NULL) { - if (isNull((const char *)&pQuery->fillVal[i], pCtx->inputType)) { - pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; - } else { // todo refactor, tVariantCreateFromBinary should handle the NULL value - if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) { - tVariantCreateFromBinary(&pCtx->param[1], (char *)&pQuery->fillVal[i], pCtx->inputBytes, pCtx->inputType); - } - } - } - } else if (functionId == TSDB_FUNC_TS_COMP) { - pCtx->param[0].i64 = vgId; - pCtx->param[0].nType = TSDB_DATA_TYPE_BIGINT; - } else if (functionId == TSDB_FUNC_TWA) { - pCtx->param[1].i64 = pQuery->window.skey; - pCtx->param[1].nType = TSDB_DATA_TYPE_BIGINT; - pCtx->param[2].i64 = pQuery->window.ekey; - pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; - } else if (functionId == TSDB_FUNC_ARITHM) { - pRuntimeEnv->sasArray[i].data = calloc(pQuery->numOfCols, POINTER_BYTES); - if (pRuntimeEnv->sasArray[i].data == NULL) { - goto _clean; - } + setTableScanFilterOperatorInfo(pRuntimeEnv->pTableScanner->info, pRuntimeEnv->proot); - pCtx->param[1].pz = (char*) &pRuntimeEnv->sasArray[i]; + if (pQuery->pExpr2 != NULL) { + pRuntimeEnv->proot = createArithOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQuery->pExpr2, pQuery->numOfExpr2); } - - if (i > 0) { - pRuntimeEnv->offset[i] = pRuntimeEnv->offset[i - 1] + pRuntimeEnv->pCtx[i - 1].outputBytes; - pRuntimeEnv->rowCellInfoOffset[i] = - pRuntimeEnv->rowCellInfoOffset[i - 1] + sizeof(SResultRowCellInfo) + pQuery->pExpr1[i - 1].interBytes; + } else { // diff/add/multiply/subtract/division + assert(pQuery->checkResultBuf == 1); + if (!onlyQueryTags(pQuery)) { + pRuntimeEnv->proot = + createArithOperatorInfo(pRuntimeEnv, pRuntimeEnv->pTableScanner, pQuery->pExpr1, pQuery->numOfOutput); + setTableScanFilterOperatorInfo(pRuntimeEnv->pTableScanner->info, pRuntimeEnv->proot); } } - *(int64_t*) pRuntimeEnv->prevRow[0] = INT64_MIN; - - // if it is group by normal column, do not set output buffer, the output buffer is pResult - // fixed output query/multi-output query for normal table - if (!pRuntimeEnv->groupbyColumn && !pRuntimeEnv->stableQuery && !QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQuery)) { - resetDefaultResInfoOutputBuf(pRuntimeEnv); + if (pQuery->limit.offset > 0) { + pRuntimeEnv->proot = createOffsetOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot); } - if (setCtxTagColumnInfo(pRuntimeEnv, pRuntimeEnv->pCtx) != TSDB_CODE_SUCCESS) { - goto _clean; + if (pQuery->limit.limit > 0) { + pRuntimeEnv->proot = createLimitOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot); } - qDebug("QInfo:%p init runtime completed", GET_QINFO_ADDR(pRuntimeEnv)); return TSDB_CODE_SUCCESS; _clean: - tfree(pRuntimeEnv->pCtx); - tfree(pRuntimeEnv->offset); - tfree(pRuntimeEnv->rowCellInfoOffset); tfree(pRuntimeEnv->sasArray); tfree(pRuntimeEnv->pResultRowHashTable); tfree(pRuntimeEnv->keyBuf); @@ -2031,74 +1813,37 @@ _clean: return TSDB_CODE_QRY_OUT_OF_MEMORY; } -static void doFreeQueryHandle(SQInfo* pQInfo) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; +static void doFreeQueryHandle(SQueryRuntimeEnv* pRuntimeEnv) { + SQuery* pQuery = pRuntimeEnv->pQuery; tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); - tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); - pRuntimeEnv->pQueryHandle = NULL; - pRuntimeEnv->pSecQueryHandle = NULL; - SMemRef* pMemRef = &pQInfo->memRef; + SMemRef* pMemRef = &pQuery->memRef; assert(pMemRef->ref == 0 && pMemRef->imem == NULL && pMemRef->mem == NULL); } - static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { - if (pRuntimeEnv->pQuery == NULL) { - return; - } - SQuery *pQuery = pRuntimeEnv->pQuery; - SQInfo* pQInfo = (SQInfo*) GET_QINFO_ADDR(pRuntimeEnv); + SQInfo* pQInfo = (SQInfo*) pRuntimeEnv->qinfo; qDebug("QInfo:%p teardown runtime env", pQInfo); - cleanupResultRowInfo(&pRuntimeEnv->resultRowInfo); - - if (isTsCompQuery(pQuery)) { - FILE *f = *(FILE **)pQuery->sdata[0]->data; - - if (f) { - fclose(f); - *(FILE **)pQuery->sdata[0]->data = NULL; - } - } - - - if (pRuntimeEnv->pCtx != NULL) { - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; - - for (int32_t j = 0; j < pCtx->numOfParams; ++j) { - tVariantDestroy(&pCtx->param[j]); - } - - tVariantDestroy(&pCtx->tag); - tfree(pCtx->tagInfo.pTagCtxList); - } - - tfree(pRuntimeEnv->pCtx); - } if (pRuntimeEnv->sasArray != NULL) { for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { tfree(pRuntimeEnv->sasArray[i].data); + tfree(pRuntimeEnv->sasArray[i].colList); } tfree(pRuntimeEnv->sasArray); } - pRuntimeEnv->pFillInfo = taosDestroyFillInfo(pRuntimeEnv->pFillInfo); - destroyResultBuf(pRuntimeEnv->pResultBuf); - doFreeQueryHandle(pQInfo); + doFreeQueryHandle(pRuntimeEnv); pRuntimeEnv->pTsBuf = tsBufDestroy(pRuntimeEnv->pTsBuf); - tfree(pRuntimeEnv->offset); tfree(pRuntimeEnv->keyBuf); - tfree(pRuntimeEnv->rowCellInfoOffset); tfree(pRuntimeEnv->prevRow); tfree(pRuntimeEnv->tagVal); @@ -2108,6 +1853,11 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { pRuntimeEnv->pool = destroyResultRowPool(pRuntimeEnv->pool); taosArrayDestroyEx(pRuntimeEnv->prevResult, freeInterResult); pRuntimeEnv->prevResult = NULL; + + taosHashCleanup(pRuntimeEnv->pTableRetrieveTsMap); + pRuntimeEnv->pTableRetrieveTsMap = NULL; + + destroyOperatorInfo(pRuntimeEnv->proot); } static bool needBuildResAfterQueryComplete(SQInfo* pQInfo) { @@ -2135,14 +1885,13 @@ bool isQueryKilled(SQInfo *pQInfo) { void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_TSC_QUERY_CANCELLED;} -static bool isFixedOutputQuery(SQueryRuntimeEnv* pRuntimeEnv) { - SQuery* pQuery = pRuntimeEnv->pQuery; +static bool isFixedOutputQuery(SQuery* pQuery) { if (QUERY_IS_INTERVAL_QUERY(pQuery)) { return false; } // Note:top/bottom query is fixed output query - if (pRuntimeEnv->topBotQuery || pRuntimeEnv->groupbyColumn) { + if (pQuery->topBotQuery || pQuery->groupbyColumn || isTsCompQuery(pQuery)) { return true; } @@ -2180,7 +1929,7 @@ bool isPointInterpoQuery(SQuery *pQuery) { } // TODO REFACTOR:MERGE WITH CLIENT-SIDE FUNCTION -static bool isSumAvgRateQuery(SQuery *pQuery) { +static UNUSED_FUNC bool isSumAvgRateQuery(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pExpr1[i].base.functionId; if (functionId == TSDB_FUNC_TS) { @@ -2297,11 +2046,11 @@ static void setScanLimitationByResultBuffer(SQuery *pQuery) { /* * todo add more parameters to check soon.. */ -bool colIdCheck(SQuery *pQuery) { +bool colIdCheck(SQuery *pQuery, void* qinfo) { // load data column information is incorrect for (int32_t i = 0; i < pQuery->numOfCols - 1; ++i) { if (pQuery->colList[i].colId == pQuery->colList[i + 1].colId) { - qError("QInfo:%p invalid data load column for query", GET_QINFO_ADDR(pQuery)); + qError("QInfo:%p invalid data load column for query", qinfo); return false; } } @@ -2333,9 +2082,10 @@ static bool onlyFirstQuery(SQuery *pQuery) { return onlyOneQueryType(pQuery, TSD static bool onlyLastQuery(SQuery *pQuery) { return onlyOneQueryType(pQuery, TSDB_FUNC_LAST, TSDB_FUNC_LAST_DST); } static void doExchangeTimeWindow(SQInfo* pQInfo, STimeWindow* win) { - size_t t = taosArrayGetSize(pQInfo->tableGroupInfo.pGroupList); + SQuery* pQuery = &pQInfo->query; + size_t t = taosArrayGetSize(pQuery->tableGroupInfo.pGroupList); for(int32_t i = 0; i < t; ++i) { - SArray* p1 = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, i); + SArray* p1 = taosArrayGetP(pQuery->tableGroupInfo.pGroupList, i); size_t len = taosArrayGetSize(p1); for(int32_t j = 0; j < len; ++j) { @@ -2381,8 +2131,7 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo if (isPointInterpoQuery(pQuery) && pQuery->interval.interval == 0) { if (!QUERY_IS_ASC_QUERY(pQuery)) { - qDebug(msg, GET_QINFO_ADDR(pQuery), "interp", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey, - pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); + qDebug(msg, pQInfo, "interp", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey, pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); } @@ -2393,7 +2142,7 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo if (pQuery->interval.interval == 0) { if (onlyFirstQuery(pQuery)) { if (!QUERY_IS_ASC_QUERY(pQuery)) { - qDebug(msg, GET_QINFO_ADDR(pQuery), "only-first", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey, + qDebug(msg, pQInfo, "only-first", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey, pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); @@ -2403,7 +2152,7 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo pQuery->order.order = TSDB_ORDER_ASC; } else if (onlyLastQuery(pQuery)) { if (QUERY_IS_ASC_QUERY(pQuery)) { - qDebug(msg, GET_QINFO_ADDR(pQuery), "only-last", pQuery->order.order, TSDB_ORDER_DESC, pQuery->window.skey, + qDebug(msg, pQInfo, "only-last", pQuery->order.order, TSDB_ORDER_DESC, pQuery->window.skey, pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); @@ -2417,7 +2166,7 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo if (stableQuery) { if (onlyFirstQuery(pQuery)) { if (!QUERY_IS_ASC_QUERY(pQuery)) { - qDebug(msg, GET_QINFO_ADDR(pQuery), "only-first stable", pQuery->order.order, TSDB_ORDER_ASC, + qDebug(msg, pQInfo, "only-first stable", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey, pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); @@ -2427,7 +2176,7 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo pQuery->order.order = TSDB_ORDER_ASC; } else if (onlyLastQuery(pQuery)) { if (QUERY_IS_ASC_QUERY(pQuery)) { - qDebug(msg, GET_QINFO_ADDR(pQuery), "only-last stable", pQuery->order.order, TSDB_ORDER_DESC, + qDebug(msg, pQInfo, "only-last stable", pQuery->order.order, TSDB_ORDER_DESC, pQuery->window.skey, pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); @@ -2440,48 +2189,29 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo } } -static int32_t getInitialPageNum(SQInfo *pQInfo) { - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - int32_t INITIAL_RESULT_ROWS_VALUE = 16; - - int32_t num = 0; - - if (isGroupbyColumn(pQuery->pGroupbyExpr)) { - num = 128; - } else if (QUERY_IS_INTERVAL_QUERY(pQuery)) { // time window query, allocate one page for each table - size_t s = pQInfo->tableqinfoGroupInfo.numOfTables; - num = (int32_t)(MAX(s, INITIAL_RESULT_ROWS_VALUE)); - } else { // for super table query, one page for each subset - num = 1; // pQInfo->pSidSet->numOfSubSet; - } - - assert(num > 0); - return num; -} - static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, int32_t* rowsize) { SQuery* pQuery = pRuntimeEnv->pQuery; int32_t MIN_ROWS_PER_PAGE = 4; - *rowsize = (int32_t)(pQuery->resultRowSize * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery)); + *rowsize = (int32_t)(pQuery->resultRowSize * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pQuery->topBotQuery, pQuery->stableQuery)); int32_t overhead = sizeof(tFilePage); // one page contains at least two rows *ps = DEFAULT_INTERN_BUF_PAGE_SIZE; while(((*rowsize) * MIN_ROWS_PER_PAGE) > (*ps) - overhead) { - *ps = (*ps << 1u); + *ps = ((*ps) << 1u); } - pRuntimeEnv->numOfRowsPerPage = ((*ps) - sizeof(tFilePage)) / (*rowsize); - assert(pRuntimeEnv->numOfRowsPerPage <= MAX_ROWS_PER_RESBUF_PAGE); +// pRuntimeEnv->numOfRowsPerPage = ((*ps) - sizeof(tFilePage)) / (*rowsize); +// assert(pRuntimeEnv->numOfRowsPerPage <= MAX_ROWS_PER_RESBUF_PAGE); } #define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR) -static bool needToLoadDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, SQLFunctionCtx *pCtx, - int32_t numOfRows) { +static bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, SQLFunctionCtx *pCtx, int32_t numOfRows) { SQuery* pQuery = pRuntimeEnv->pQuery; - if (pDataStatis == NULL || (pQuery->numOfFilterCols == 0 && (!pRuntimeEnv->topBotQuery))) { + + if (pDataStatis == NULL || pQuery->numOfFilterCols == 0) { return true; } @@ -2540,15 +2270,6 @@ static bool needToLoadDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis *pDat } } - if (pRuntimeEnv->topBotQuery) { - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionId = pQuery->pExpr1[i].base.functionId; - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { - return topbot_datablock_filter(&pCtx[i], functionId, (char *)&pDataStatis[i].min, (char *)&pDataStatis[i].max); - } - } - } - return false; } @@ -2601,86 +2322,323 @@ static bool overlapWithTimeWindow(SQuery* pQuery, SDataBlockInfo* pBlockInfo) { return false; } -int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo * pWindowResInfo, void* pQueryHandle, SDataBlockInfo* pBlockInfo, SDataStatis **pStatis, SArray** pDataBlock, uint32_t* status) { - *status = BLK_DATA_NO_NEEDED; - - SQuery *pQuery = pRuntimeEnv->pQuery; - int64_t groupId = pQuery->current->groupIndex; +static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, TSKEY key, bool ascQuery) { + STSElem elem = tsBufGetElem(pRuntimeEnv->pTsBuf); - SQueryCostInfo* pCost = &pRuntimeEnv->summary; +#if defined(_DEBUG_VIEW) + printf("elem in comp ts file:%" PRId64 ", key:%" PRId64 ", tag:%"PRIu64", query order:%d, ts order:%d, traverse:%d, index:%d\n", + elem.ts, key, elem.tag.i64, pQuery->order.order, pRuntimeEnv->pTsBuf->tsOrder, + pRuntimeEnv->pTsBuf->cur.order, pRuntimeEnv->pTsBuf->cur.tsIndex); +#endif - if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf > 0) { - *status = BLK_DATA_ALL_NEEDED; - } else { // check if this data block is required to load - // Calculate all time windows that are overlapping or contain current data block. - // If current data block is contained by all possible time window, do not load current data block. - if (QUERY_IS_INTERVAL_QUERY(pQuery) && overlapWithTimeWindow(pQuery, pBlockInfo)) { - *status = BLK_DATA_ALL_NEEDED; + if (ascQuery) { + if (key < elem.ts) { + return TS_JOIN_TS_NOT_EQUALS; + } else if (key > elem.ts) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_INCONSISTAN); } - - if ((*status) != BLK_DATA_ALL_NEEDED) { - // the pCtx[i] result is belonged to previous time window since the outputBuf has not been set yet, - // the filter result may be incorrect. So in case of interval query, we need to set the correct time output buffer - if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - SResultRow* pResult = NULL; - - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); - - TSKEY k = QUERY_IS_ASC_QUERY(pQuery)? pBlockInfo->window.skey:pBlockInfo->window.ekey; - STimeWindow win = getActiveTimeWindow(pWindowResInfo, k, pQuery); - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId) != TSDB_CODE_SUCCESS) { - // todo handle error in set result for timewindow - } + } else { + if (key > elem.ts) { + return TS_JOIN_TS_NOT_EQUALS; + } else if (key < elem.ts) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_INCONSISTAN); + } + } + + return TS_JOIN_TS_EQUAL; +} + +void filterRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, + SSDataBlock* pBlock, STSBuf* pTsBuf, bool ascQuery) { + int32_t numOfRows = pBlock->info.rows; + + int8_t *p = calloc(numOfRows, sizeof(int8_t)); + bool all = true; + + if (pTsBuf != NULL) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); + + TSKEY* k = (TSKEY*) pColInfoData->pData; + for (int32_t i = 0; i < numOfRows; ++i) { + int32_t offset = ascQuery? i:(numOfRows - i - 1); + int32_t ret = doTSJoinFilter(pRuntimeEnv, k[offset], ascQuery); + if (ret == TS_JOIN_TAG_NOT_EQUALS) { + break; + } else if (ret == TS_JOIN_TS_NOT_EQUALS) { + all = false; + continue; + } else { + assert(ret == TS_JOIN_TS_EQUAL); + p[offset] = true; } - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SSqlFuncMsg* pSqlFunc = &pQuery->pExpr1[i].base; + if (!tsBufNextPos(pRuntimeEnv->pTsBuf)) { + break; + } + } + } else { + for (int32_t i = 0; i < numOfRows; ++i) { + bool qualified = false; + + for (int32_t k = 0; k < numOfFilterCols; ++k) { + char* pElem = (char*)pFilterInfo[k].pData + pFilterInfo[k].info.bytes * i; + + qualified = false; + for (int32_t j = 0; j < pFilterInfo[k].numOfFilters; ++j) { + SColumnFilterElem* pFilterElem = &pFilterInfo[k].pFilters[j]; + + bool isnull = isNull(pElem, pFilterInfo[k].info.type); + if (isnull) { + if (pFilterElem->fp == isNullOperator) { + qualified = true; + break; + } else { + continue; + } + } else { + if (pFilterElem->fp == notNullOperator) { + qualified = true; + break; + } else if (pFilterElem->fp == isNullOperator) { + continue; + } + } + + if (pFilterElem->fp(pFilterElem, pElem, pElem, pFilterInfo[k].info.type)) { + qualified = true; + break; + } + } - int32_t functionId = pSqlFunc->functionId; - int32_t colId = pSqlFunc->colInfo.colId; - (*status) |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], pBlockInfo->window.skey, pBlockInfo->window.ekey, colId); - if (((*status) & BLK_DATA_ALL_NEEDED) == BLK_DATA_ALL_NEEDED) { + if (!qualified) { break; } } + + p[i] = qualified ? 1 : 0; + if (!qualified) { + all = false; + } + } + } + + if (!all) { + int32_t start = 0; + int32_t len = 0; + for (int32_t j = 0; j < numOfRows; ++j) { + if (p[j] == 1) { + len++; + } else { + if (len > 0) { + int32_t cstart = j - len; + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData *pColumnInfoData = taosArrayGet(pBlock->pDataBlock, i); + + int16_t bytes = pColumnInfoData->info.bytes; + memmove(((char*)pColumnInfoData->pData) + start * bytes, pColumnInfoData->pData + cstart * bytes, len * bytes); + } + + start += len; + len = 0; + } + } + } + + if (len > 0) { + int32_t cstart = numOfRows - len; + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData *pColumnInfoData = taosArrayGet(pBlock->pDataBlock, i); + + int16_t bytes = pColumnInfoData->info.bytes; + memmove(pColumnInfoData->pData + start * bytes, pColumnInfoData->pData + cstart * bytes, len * bytes); + } + + start += len; + len = 0; + } + + pBlock->info.rows = start; + pBlock->pBlockStatis = NULL; // clean the block statistics info + + if (start > 0) { + SColumnInfoData* pColumnInfoData = taosArrayGet(pBlock->pDataBlock, 0); + assert(pColumnInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP && + pColumnInfoData->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX); + + pBlock->info.window.skey = *(int64_t*)pColumnInfoData->pData; + pBlock->info.window.ekey = *(int64_t*)(pColumnInfoData->pData + TSDB_KEYSIZE * (start - 1)); + } + } + + tfree(p); +} + +static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId); +static void doSetTagValueInParam(void* pTable, int32_t tagColId, tVariant *tag, int16_t type, int16_t bytes); + +static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) { + SQLFunctionCtx* pCtx = pTableScanInfo->pCtx; + uint32_t status = BLK_DATA_NO_NEEDED; + + int32_t numOfOutput = pTableScanInfo->numOfOutput; + for (int32_t i = 0; i < numOfOutput; ++i) { + int32_t functionId = pCtx[i].functionId; + int32_t colId = pTableScanInfo->pExpr[i].base.colInfo.colId; + + // group by + first/last should not apply the first/last block filter + status |= aAggs[functionId].dataReqFunc(&pTableScanInfo->pCtx[i], &pBlock->info.window, colId); + if ((status & BLK_DATA_ALL_NEEDED) == BLK_DATA_ALL_NEEDED) { + return status; + } + } + + return status; +} + +static void doSetFilterColumnInfo(SQuery* pQuery, SSDataBlock* pBlock) { + if (pQuery->numOfFilterCols > 0 && pQuery->pFilterInfo[0].pData != NULL) { + return; + } + + // set the initial static data value filter expression + for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { + for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, j); + + if (pQuery->pFilterInfo[i].info.colId == pColInfo->info.colId) { + pQuery->pFilterInfo[i].pData = pColInfo->pData; + break; + } + } + } +} + +int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, + uint32_t* status) { + *status = BLK_DATA_NO_NEEDED; + pBlock->pDataBlock = NULL; + pBlock->pBlockStatis = NULL; + + SQuery* pQuery = pRuntimeEnv->pQuery; + int64_t groupId = pQuery->current->groupIndex; + bool ascQuery = QUERY_IS_ASC_QUERY(pQuery); + + SQInfo* pQInfo = pRuntimeEnv->qinfo; + SQueryCostInfo* pCost = &pQInfo->summary; + + if (pRuntimeEnv->pTsBuf != NULL) { + (*status) = BLK_DATA_ALL_NEEDED; + + if (pQuery->stableQuery) { // todo refactor + SExprInfo* pExprInfo = &pTableScanInfo->pExpr[0]; + int16_t tagId = (int16_t)pExprInfo->base.arg->argValue.i64; + SColumnInfo* pColInfo = doGetTagColumnInfoById(pQuery->tagColList, pQuery->numOfTags, tagId); + + // compare tag first + tVariant t = {0}; + doSetTagValueInParam(pQuery->current->pTable, tagId, &t, pColInfo->type, pColInfo->bytes); + setTimestampListJoinInfo(pRuntimeEnv, &t, pQuery->current); + + STSElem elem = tsBufGetElem(pRuntimeEnv->pTsBuf); + if (!tsBufIsValidElem(&elem) || (tsBufIsValidElem(&elem) && (tVariantCompare(&t, elem.tag) != 0))) { + (*status) = BLK_DATA_DISCARD; + return TSDB_CODE_SUCCESS; + } + } + } + + // Calculate all time windows that are overlapping or contain current data block. + // If current data block is contained by all possible time window, do not load current data block. + if (pQuery->numOfFilterCols > 0 || pQuery->groupbyColumn || pQuery->sw.gap > 0 || + (QUERY_IS_INTERVAL_QUERY(pQuery) && overlapWithTimeWindow(pQuery, &pBlock->info))) { + (*status) = BLK_DATA_ALL_NEEDED; + } + + // check if this data block is required to load + if ((*status) != BLK_DATA_ALL_NEEDED) { + // the pCtx[i] result is belonged to previous time window since the outputBuf has not been set yet, + // the filter result may be incorrect. So in case of interval query, we need to set the correct time output buffer + if (QUERY_IS_INTERVAL_QUERY(pQuery)) { + SResultRow* pResult = NULL; + + bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey; + + STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQuery); + if (setWindowOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, &win, masterScan, &pResult, groupId, + pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, + pTableScanInfo->rowCellInfoOffset) != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } else if (pQuery->stableQuery && (!isTsCompQuery(pQuery))) { // stable aggregate, not interval aggregate or normal column aggregate + doSetTableGroupOutputBuf(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pTableScanInfo->pCtx, + pTableScanInfo->rowCellInfoOffset, pTableScanInfo->numOfOutput, + pQuery->current->groupIndex); } + + (*status) = doFilterByBlockTimeWindow(pTableScanInfo, pBlock); } + SDataBlockInfo* pBlockInfo = &pBlock->info; if ((*status) == BLK_DATA_NO_NEEDED) { - qDebug("QInfo:%p data block discard, brange:%"PRId64 "-%"PRId64", rows:%d", GET_QINFO_ADDR(pRuntimeEnv), - pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + qDebug("QInfo:%p data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo, pBlockInfo->window.skey, + pBlockInfo->window.ekey, pBlockInfo->rows); pCost->discardBlocks += 1; } else if ((*status) == BLK_DATA_STATIS_NEEDED) { - // this function never returns error? - tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis); pCost->loadBlockStatis += 1; + tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockStatis); - if (*pStatis == NULL) { // data block statistics does not exist, load data block - *pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); - pCost->totalCheckedRows += pBlockInfo->rows; + if (pBlock->pBlockStatis == NULL) { // data block statistics does not exist, load data block + pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); + pCost->totalCheckedRows += pBlock->info.rows; } } else { assert((*status) == BLK_DATA_ALL_NEEDED); // load the data block statistics to perform further filter pCost->loadBlockStatis += 1; - tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis); + tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockStatis); + + if (pQuery->topBotQuery && pBlock->pBlockStatis != NULL) { + bool load = false; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functionId = pTableScanInfo->pCtx[i].functionId; + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { + load = topbot_datablock_filter(&pTableScanInfo->pCtx[i], (char*)&(pBlock->pBlockStatis[i].min), + (char*)&(pBlock->pBlockStatis[i].max)); + if (!load) { // current block has been discard due to filter applied + pCost->discardBlocks += 1; + qDebug("QInfo:%p data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo, + pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + (*status) = BLK_DATA_DISCARD; + return TSDB_CODE_SUCCESS; + } + } + } + } - if (!needToLoadDataBlock(pRuntimeEnv, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->rows)) { - // current block has been discard due to filter applied + // current block has been discard due to filter applied + if (!doFilterByBlockStatistics(pRuntimeEnv, pBlock->pBlockStatis, pTableScanInfo->pCtx, pBlockInfo->rows)) { pCost->discardBlocks += 1; - qDebug("QInfo:%p data block discard, brange:%"PRId64 "-%"PRId64", rows:%d", GET_QINFO_ADDR(pRuntimeEnv), - pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + qDebug("QInfo:%p data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo, pBlockInfo->window.skey, + pBlockInfo->window.ekey, pBlockInfo->rows); (*status) = BLK_DATA_DISCARD; + return TSDB_CODE_SUCCESS; } pCost->totalCheckedRows += pBlockInfo->rows; pCost->loadBlocks += 1; - *pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); - if (*pDataBlock == NULL) { + pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); + if (pBlock->pDataBlock == NULL) { return terrno; } + + doSetFilterColumnInfo(pQuery, pBlock); + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL) { + filterRowsInDataBlock(pRuntimeEnv, pQuery->pFilterInfo, pQuery->numOfFilterCols, pBlock, pRuntimeEnv->pTsBuf, + ascQuery); + } } return TSDB_CODE_SUCCESS; @@ -2749,148 +2707,6 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { return midPos; } -static void expandBuffer(SQueryRuntimeEnv* pRuntimeEnv, int32_t newSize, void* qinfo) { - SQuery* pQuery = pRuntimeEnv->pQuery; - SResultRec *pRec = &pQuery->rec; - - assert(newSize > 0); - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t bytes = pQuery->pExpr1[i].bytes; - - char *tmp = realloc(pQuery->sdata[i], bytes * newSize + sizeof(tFilePage)); - if (tmp == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } else { - memset(tmp + sizeof(tFilePage) + bytes * pRec->rows, 0, (size_t)((newSize - pRec->rows) * bytes)); - pQuery->sdata[i] = (tFilePage *)tmp; - } - } - - pRec->capacity = newSize; - qDebug("QInfo:%p realloc output buffer, new size: %d rows, old:%" PRId64 ", remain:%" PRId64, qinfo, newSize, - pRec->capacity, newSize - pRec->rows); -} - -static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, int32_t numOfRows) { - // in case of prj/diff query, ensure the output buffer is sufficient to accommodate the results of current block - SQuery* pQuery = pRuntimeEnv->pQuery; - if (!QUERY_IS_INTERVAL_QUERY(pQuery) && !pRuntimeEnv->groupbyColumn && !isFixedOutputQuery(pRuntimeEnv) && !isTsCompQuery(pQuery)) { - SResultRec *pRec = &pQuery->rec; - - int32_t remain = (int32_t)(pRec->capacity - pRec->rows); - if (remain < numOfRows) { - int32_t newSize = (int32_t)(pRec->capacity + (numOfRows - remain)); - expandBuffer(pRuntimeEnv, newSize, GET_QINFO_ADDR(pRuntimeEnv)); - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t bytes = pQuery->pExpr1[i].bytes; - - // set the pCtx output buffer position - pRuntimeEnv->pCtx[i].pOutput = pQuery->sdata[i]->data + pRec->rows * bytes; - - int32_t functionId = pQuery->pExpr1[i].base.functionId; - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - pRuntimeEnv->pCtx[i].ptsOutputBuf = pRuntimeEnv->pCtx[0].pOutput; - } - } - } - } -} - -static void doSetInitialTimewindow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pBlockInfo) { - SQuery* pQuery = pRuntimeEnv->pQuery; - - if (QUERY_IS_INTERVAL_QUERY(pQuery) && pRuntimeEnv->resultRowInfo.prevSKey == TSKEY_INITIAL_VAL) { - STimeWindow w = TSWINDOW_INITIALIZER; - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; - - if (QUERY_IS_ASC_QUERY(pQuery)) { - getAlignQueryTimeWindow(pQuery, pBlockInfo->window.skey, pBlockInfo->window.skey, pQuery->window.ekey, &w); - pWindowResInfo->prevSKey = w.skey; - } else { // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp - getAlignQueryTimeWindow(pQuery, pBlockInfo->window.ekey, pQuery->window.ekey, pBlockInfo->window.ekey, &w); - pWindowResInfo->prevSKey = w.skey; - } - } -} - -static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { - SQuery *pQuery = pRuntimeEnv->pQuery; - STableQueryInfo* pTableQueryInfo = pQuery->current; - SQueryCostInfo* summary = &pRuntimeEnv->summary; - - qDebug("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", lastkey:%" PRId64 ", order:%d", - GET_QINFO_ADDR(pRuntimeEnv), pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, pTableQueryInfo->lastKey, - pQuery->order.order); - - TsdbQueryHandleT pQueryHandle = IS_MASTER_SCAN(pRuntimeEnv)? pRuntimeEnv->pQueryHandle : pRuntimeEnv->pSecQueryHandle; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - - SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; - while (tsdbNextDataBlock(pQueryHandle)) { - summary->totalBlocks += 1; - - if (IS_MASTER_SCAN(pRuntimeEnv)) { - pQuery->numOfCheckedBlocks += 1; - } - - if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { - longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); - } - - tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); - doSetInitialTimewindow(pRuntimeEnv, &blockInfo); - - // in case of prj/diff query, ensure the output buffer is sufficient to accommodate the results of current block - ensureOutputBuffer(pRuntimeEnv, blockInfo.rows); - - SDataStatis *pStatis = NULL; - SArray * pDataBlock = NULL; - uint32_t status = 0; - - int32_t ret = loadDataBlockOnDemand(pRuntimeEnv, &pRuntimeEnv->resultRowInfo, pQueryHandle, &blockInfo, &pStatis, &pDataBlock, &status); - if (ret != TSDB_CODE_SUCCESS) { - break; - } - - if (status == BLK_DATA_DISCARD) { - pQuery->current->lastKey = - QUERY_IS_ASC_QUERY(pQuery) ? blockInfo.window.ekey + step : blockInfo.window.skey + step; - continue; - } - - // query start position can not move into tableApplyFunctionsOnBlock due to limit/offset condition - pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : blockInfo.rows - 1; - int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, pDataBlock); - - summary->totalRows += blockInfo.rows; - qDebug("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%"PRId64, GET_QINFO_ADDR(pRuntimeEnv), - blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes, pQuery->current->lastKey); - - // while the output buffer is full or limit/offset is applied, query may be paused here - if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL | QUERY_COMPLETED)) { - break; - } - } - - if (terrno != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, terrno); - } - - // if the result buffer is not full, set the query complete - if (!Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { - setQueryStatus(pQuery, QUERY_COMPLETED); - } - - if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - closeAllResultRows(&pRuntimeEnv->resultRowInfo); - pRuntimeEnv->resultRowInfo.curIndex = pRuntimeEnv->resultRowInfo.size - 1; // point to the last time window - } - - return 0; -} - /* * set tag value in SQLFunctionCtx * e.g.,tag information into input buffer @@ -2898,33 +2714,23 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { static void doSetTagValueInParam(void* pTable, int32_t tagColId, tVariant *tag, int16_t type, int16_t bytes) { tVariantDestroy(tag); + char* val = NULL; if (tagColId == TSDB_TBNAME_COLUMN_INDEX) { - char* val = tsdbGetTableName(pTable); + val = tsdbGetTableName(pTable); assert(val != NULL); - - tVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), TSDB_DATA_TYPE_BINARY); } else { - char* val = tsdbGetTableTagVal(pTable, tagColId, type, bytes); - if (val == NULL) { - tag->nType = TSDB_DATA_TYPE_NULL; - return; - } - - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - if (isNull(val, type)) { - tag->nType = TSDB_DATA_TYPE_NULL; - return; - } + val = tsdbGetTableTagVal(pTable, tagColId, type, bytes); + } - tVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), type); - } else { - if (isNull(val, type)) { - tag->nType = TSDB_DATA_TYPE_NULL; - return; - } + if (val == NULL || isNull(val, type)) { + tag->nType = TSDB_DATA_TYPE_NULL; + return; + } - tVariantCreateFromBinary(tag, val, bytes, type); - } + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + tVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), type); + } else { + tVariantCreateFromBinary(tag, val, bytes, type); } } @@ -2940,24 +2746,28 @@ static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t num return NULL; } -void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable) { - SQuery *pQuery = pRuntimeEnv->pQuery; - SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); +void setTagValue(SOperatorInfo* pOperatorInfo, void *pTable, SQLFunctionCtx* pCtx, int32_t numOfOutput) { + SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - SExprInfo *pExprInfo = &pQuery->pExpr1[0]; - if (pQuery->numOfOutput == 1 && pExprInfo->base.functionId == TSDB_FUNC_TS_COMP && pRuntimeEnv->stableQuery) { + SExprInfo *pExpr = pOperatorInfo->pExpr; + SQuery *pQuery = pRuntimeEnv->pQuery; + + SExprInfo* pExprInfo = &pExpr[0]; + if (pQuery->numOfOutput == 1 && pExprInfo->base.functionId == TSDB_FUNC_TS_COMP && pQuery->stableQuery) { assert(pExprInfo->base.numOfParams == 1); - int16_t tagColId = (int16_t)pExprInfo->base.arg->argValue.i64; + int16_t tagColId = (int16_t)pExprInfo->base.arg->argValue.i64; SColumnInfo* pColInfo = doGetTagColumnInfoById(pQuery->tagColList, pQuery->numOfTags, tagColId); - doSetTagValueInParam(pTable, tagColId, &pRuntimeEnv->pCtx[0].tag, pColInfo->type, pColInfo->bytes); + doSetTagValueInParam(pTable, tagColId, &pCtx[0].tag, pColInfo->type, pColInfo->bytes); + return; } else { // set tag value, by which the results are aggregated. int32_t offset = 0; memset(pRuntimeEnv->tagVal, 0, pQuery->tagLen); - for (int32_t idx = 0; idx < pQuery->numOfOutput; ++idx) { - SExprInfo* pLocalExprInfo = &pQuery->pExpr1[idx]; + + for (int32_t idx = 0; idx < numOfOutput; ++idx) { + SExprInfo* pLocalExprInfo = &pExpr[idx]; // ts_comp column required the tag value for join filter if (!TSDB_COL_IS_TAG(pLocalExprInfo->base.colInfo.flag)) { @@ -2965,39 +2775,28 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable) { } // todo use tag column index to optimize performance - doSetTagValueInParam(pTable, pLocalExprInfo->base.colInfo.colId, &pRuntimeEnv->pCtx[idx].tag, - pLocalExprInfo->type, pLocalExprInfo->bytes); + doSetTagValueInParam(pTable, pLocalExprInfo->base.colInfo.colId, &pCtx[idx].tag, pLocalExprInfo->type, + pLocalExprInfo->bytes); if (IS_NUMERIC_TYPE(pLocalExprInfo->type) || pLocalExprInfo->type == TSDB_DATA_TYPE_BOOL) { - memcpy(pRuntimeEnv->tagVal + offset, &pRuntimeEnv->pCtx[idx].tag.i64, pLocalExprInfo->bytes); + memcpy(pRuntimeEnv->tagVal + offset, &pCtx[idx].tag.i64, pLocalExprInfo->bytes); } else { - memcpy(pRuntimeEnv->tagVal + offset, pRuntimeEnv->pCtx[idx].tag.pz, pRuntimeEnv->pCtx[idx].tag.nLen); + memcpy(pRuntimeEnv->tagVal + offset, pCtx[idx].tag.pz, pCtx[idx].tag.nLen); } offset += pLocalExprInfo->bytes; } - // set the join tag for first column - SSqlFuncMsg *pFuncMsg = &pExprInfo->base; - if ((pFuncMsg->functionId == TSDB_FUNC_TS || pFuncMsg->functionId == TSDB_FUNC_PRJ) && pRuntimeEnv->pTsBuf != NULL && - pFuncMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - assert(pFuncMsg->numOfParams == 1); - - int16_t tagColId = (int16_t)pExprInfo->base.arg->argValue.i64; - SColumnInfo *pColInfo = doGetTagColumnInfoById(pQuery->tagColList, pQuery->numOfTags, tagColId); - - doSetTagValueInParam(pTable, tagColId, &pRuntimeEnv->pCtx[0].tag, pColInfo->type, pColInfo->bytes); - - int16_t tagType = pRuntimeEnv->pCtx[0].tag.nType; - if (tagType == TSDB_DATA_TYPE_BINARY || tagType == TSDB_DATA_TYPE_NCHAR) { - qDebug("QInfo:%p set tag value for join comparison, colId:%" PRId64 ", val:%s", pQInfo, - pExprInfo->base.arg->argValue.i64, pRuntimeEnv->pCtx[0].tag.pz); - } else { - qDebug("QInfo:%p set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64, pQInfo, - pExprInfo->base.arg->argValue.i64, pRuntimeEnv->pCtx[0].tag.i64); - } + //todo : use index to avoid iterator all possible output columns + if (pQuery->stableQuery && pQuery->stabledev && (pRuntimeEnv->prevResult != NULL)) { + setParamForStableStddev(pRuntimeEnv, pCtx, numOfOutput, pExprInfo); } } + + // set the tsBuf start position before check each data block + if (pRuntimeEnv->pTsBuf != NULL) { + setCtxTagForJoin(pRuntimeEnv, &pCtx[0], pExprInfo, pTable); + } } static UNUSED_FUNC void printBinaryData(int32_t functionId, char *data, int32_t srcDataType) { @@ -3103,35 +2902,36 @@ void UNUSED_FUNC displayInterResult(tFilePage **pdata, SQueryRuntimeEnv* pRuntim } } -static int32_t doCopyToSData(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType); +void copyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, int32_t threshold, SSDataBlock* pBlock, int32_t* offset) { + SGroupResInfo* pGroupResInfo = &pRuntimeEnv->groupResInfo; + pBlock->info.rows = 0; -void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { - SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; - - while(pGroupResInfo->currentGroup < pGroupResInfo->totalGroup) { + int32_t code = TSDB_CODE_SUCCESS; + while (pGroupResInfo->currentGroup < pGroupResInfo->totalGroup) { // all results in current group have been returned to client, try next group if ((pGroupResInfo->pRows == NULL) || taosArrayGetSize(pGroupResInfo->pRows) == 0) { assert(pGroupResInfo->index == 0); - if ((pQInfo->code = mergeIntoGroupResult(&pQInfo->groupResInfo, pQInfo)) != TSDB_CODE_SUCCESS) { + if ((code = mergeIntoGroupResult(&pRuntimeEnv->groupResInfo, pRuntimeEnv, offset)) != TSDB_CODE_SUCCESS) { return; } } - pQuery->rec.rows = doCopyToSData(&pQInfo->runtimeEnv, pGroupResInfo, TSDB_ORDER_ASC); + doCopyToSDataBlock(pRuntimeEnv, pGroupResInfo, TSDB_ORDER_ASC, pBlock); // current data are all dumped to result buffer, clear it - if (!hasRemainData(pGroupResInfo)) { + if (!hasRemainDataInCurrentGroup(pGroupResInfo)) { cleanupGroupResInfo(pGroupResInfo); if (!incNextGroup(pGroupResInfo)) { - SET_STABLE_QUERY_OVER(pQInfo); + break; } } - // enough results in data buffer, return - if (pQuery->rec.rows >= pQuery->rec.threshold) { - break; + // enough results in data buffer, return + if (pBlock->info.rows >= threshold) { + break; + } } - } + } static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo *pTableQueryInfo) { @@ -3145,71 +2945,17 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * SWITCH_ORDER(pTableQueryInfo->cur.order); pTableQueryInfo->cur.vgroupIndex = -1; - // set the index at the end of time window + // set the index to be the end slot of result rows array pTableQueryInfo->resInfo.curIndex = pTableQueryInfo->resInfo.size - 1; } -static void disableFuncInReverseScanImpl(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo *pWindowResInfo, int32_t order) { +static void setupQueryRangeForReverseScan(SQueryRuntimeEnv* pRuntimeEnv) { SQuery* pQuery = pRuntimeEnv->pQuery; - for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - bool closed = getResultRowStatus(pWindowResInfo, i); - if (!closed) { - continue; - } - - SResultRow *pRow = getResultRow(pWindowResInfo, i); - - // open/close the specified query for each group result - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int32_t functId = pQuery->pExpr1[j].base.functionId; - SResultRowCellInfo* pInfo = getResultCell(pRuntimeEnv, pRow, j); - - if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_ASC) || - ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSDB_ORDER_DESC)) { - pInfo->complete = false; - } else if (functId != TSDB_FUNC_TS && functId != TSDB_FUNC_TAG) { - pInfo->complete = true; - } - } - } -} - -void disableFuncInReverseScan(SQInfo *pQInfo) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery *pQuery = pRuntimeEnv->pQuery; - int32_t order = pQuery->order.order; - - // group by normal columns and interval query on normal table - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; - if (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery)) { - disableFuncInReverseScanImpl(pRuntimeEnv, pWindowResInfo, order); - } else { // for simple result of table query, - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { // todo refactor - int32_t functId = pQuery->pExpr1[j].base.functionId; - - SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[j]; - if (pCtx->resultInfo == NULL) { - continue; // resultInfo is NULL, means no data checked in previous scan - } - - if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_ASC) || - ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSDB_ORDER_DESC)) { - pCtx->resultInfo->complete = false; - } else if (functId != TSDB_FUNC_TS && functId != TSDB_FUNC_TAG) { - pCtx->resultInfo->complete = true; - } - } - } -} - -static void setupQueryRangeForReverseScan(SQInfo* pQInfo) { - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); - + int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pRuntimeEnv)); for(int32_t i = 0; i < numOfGroups; ++i) { - SArray *group = GET_TABLEGROUP(pQInfo, i); - SArray *tableKeyGroup = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, i); + SArray *group = GET_TABLEGROUP(pRuntimeEnv, i); + SArray *tableKeyGroup = taosArrayGetP(pQuery->tableGroupInfo.pGroupList, i); size_t t = taosArrayGetSize(group); for (int32_t j = 0; j < t; ++j) { @@ -3226,209 +2972,117 @@ static void setupQueryRangeForReverseScan(SQInfo* pQInfo) { } } -void switchCtxOrder(SQueryRuntimeEnv *pRuntimeEnv) { - SQuery *pQuery = pRuntimeEnv->pQuery; - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SWITCH_ORDER(pRuntimeEnv->pCtx[i].order); +void switchCtxOrder(SQLFunctionCtx* pCtx, int32_t numOfOutput) { + for (int32_t i = 0; i < numOfOutput; ++i) { + SWITCH_ORDER(pCtx[i].order); } } int32_t initResultRow(SResultRow *pResultRow) { pResultRow->pCellInfo = (SResultRowCellInfo*)((char*)pResultRow + sizeof(SResultRow)); pResultRow->pageId = -1; - pResultRow->rowId = -1; + pResultRow->offset = -1; return TSDB_CODE_SUCCESS; } -void resetDefaultResInfoOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { - SQuery *pQuery = pRuntimeEnv->pQuery; +/* + * The start of each column SResultRowCellInfo is denote by RowCellInfoOffset. + * Note that in case of top/bottom query, the whole multiple rows of result is treated as only one row of results. + * +------------+-----------------result column 1-----------+-----------------result column 2-----------+ + * + SResultRow | SResultRowCellInfo | intermediate buffer1 | SResultRowCellInfo | intermediate buffer 2| + * +------------+-------------------------------------------+-------------------------------------------+ + * offset[0] offset[1] + */ +void setDefaultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, int64_t uid) { + SQLFunctionCtx* pCtx = pInfo->pCtx; + SSDataBlock* pDataBlock = pInfo->pRes; + int32_t* rowCellInfoOffset = pInfo->rowCellInfoOffset; + SResultRowInfo* pResultRowInfo = &pInfo->resultRowInfo; int32_t tid = 0; - int64_t uid = 0; - SResultRow* pRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->resultRowInfo, (char *)&tid, sizeof(tid), true, uid); + SResultRow* pRow = doPrepareResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char *)&tid, sizeof(tid), true, uid); - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; - pCtx->pOutput = pQuery->sdata[i]->data; + for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData* pData = taosArrayGet(pDataBlock->pDataBlock, i); /* * set the output buffer information and intermediate buffer * not all queries require the interResultBuf, such as COUNT/TAGPRJ/PRJ/TAG etc. */ - SResultRowCellInfo* pCellInfo = getResultCell(pRuntimeEnv, pRow, i); + SResultRowCellInfo* pCellInfo = getResultCell(pRow, i, rowCellInfoOffset); RESET_RESULT_INFO(pCellInfo); - pCtx->resultInfo = pCellInfo; + + pCtx[i].resultInfo = pCellInfo; + pCtx[i].pOutput = pData->pData; + assert(pCtx[i].pOutput != NULL); // set the timestamp output buffer for top/bottom/diff query - int32_t functionId = pQuery->pExpr1[i].base.functionId; + int32_t functionId = pCtx[i].functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].pOutput; + pCtx[i].ptsOutputBuf = pCtx[0].pOutput; } - - memset(pQuery->sdata[i]->data, 0, (size_t)(pQuery->pExpr1[i].bytes * pQuery->rec.capacity)); } - initCtxOutputBuf(pRuntimeEnv); + initCtxOutputBuffer(pCtx, pDataBlock->info.numOfCols); } -void forwardCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, int64_t output) { - SQuery *pQuery = pRuntimeEnv->pQuery; +void updateOutputBuf(SArithOperatorInfo* pInfo, int32_t numOfInputRows) { + SOptrBasicInfo* pBInfo = &pInfo->binfo; + SSDataBlock* pDataBlock = pBInfo->pRes; - // reset the execution contexts - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int32_t functionId = pQuery->pExpr1[j].base.functionId; - assert(functionId != TSDB_FUNC_DIFF); + int32_t newSize = pDataBlock->info.rows + numOfInputRows; + if (pInfo->bufCapacity < newSize) { + for(int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); + char* p = realloc(pColInfo->pData, newSize * pColInfo->info.bytes); + if (p != NULL) { + pColInfo->pData = p; - // set next output position - if (IS_OUTER_FORWARD(aAggs[functionId].status)) { - pRuntimeEnv->pCtx[j].pOutput += pRuntimeEnv->pCtx[j].outputBytes * output; - } - - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { - /* - * NOTE: for top/bottom query, the value of first column of output (timestamp) are assigned - * in the procedure of top/bottom routine - * the output buffer in top/bottom routine is ptsOutputBuf, so we need to forward the output buffer - * - * diff function is handled in multi-output function - */ - pRuntimeEnv->pCtx[j].ptsOutputBuf = (char*)pRuntimeEnv->pCtx[j].ptsOutputBuf + TSDB_KEYSIZE * output; + // it starts from the tail of the previously generated results. + pBInfo->pCtx[i].pOutput = pColInfo->pData; + pInfo->bufCapacity = newSize; + } else { + // longjmp + } } - - RESET_RESULT_INFO(pRuntimeEnv->pCtx[j].resultInfo); } -} -void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int32_t functionId = pQuery->pExpr1[j].base.functionId; - pRuntimeEnv->pCtx[j].currentStage = 0; + for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); + pBInfo->pCtx[i].pOutput = pColInfo->pData + pColInfo->info.bytes * pDataBlock->info.rows; - SResultRowCellInfo* pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); - if (pResInfo->initialized) { - continue; + // re-estabilish output buffer pointer. + int32_t functionId = pBInfo->pCtx[i].functionId; + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { + pBInfo->pCtx[i].ptsOutputBuf = pBInfo->pCtx[0].pOutput; } - - aAggs[functionId].init(&pRuntimeEnv->pCtx[j]); } } -void skipResults(SQueryRuntimeEnv *pRuntimeEnv) { - SQuery *pQuery = pRuntimeEnv->pQuery; - if (pQuery->rec.rows == 0 || pQuery->limit.offset == 0) { - return; - } - - if (pQuery->rec.rows <= pQuery->limit.offset) { - qDebug("QInfo:%p skip rows:%" PRId64 ", new offset:%" PRIu64, GET_QINFO_ADDR(pRuntimeEnv), pQuery->rec.rows, - pQuery->limit.offset - pQuery->rec.rows); - - pQuery->limit.offset -= pQuery->rec.rows; - pQuery->rec.rows = 0; - - resetDefaultResInfoOutputBuf(pRuntimeEnv); - - // clear the buffer full flag if exists - CLEAR_QUERY_STATUS(pQuery, QUERY_RESBUF_FULL); - } else { - int64_t numOfSkip = pQuery->limit.offset; - pQuery->rec.rows -= numOfSkip; - pQuery->limit.offset = 0; - - qDebug("QInfo:%p skip row:%"PRId64", new offset:%d, numOfRows remain:%" PRIu64, GET_QINFO_ADDR(pRuntimeEnv), numOfSkip, - 0, pQuery->rec.rows); - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionId = pQuery->pExpr1[i].base.functionId; - int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; - - memmove(pQuery->sdata[i]->data, (char*)pQuery->sdata[i]->data + bytes * numOfSkip, (size_t)(pQuery->rec.rows * bytes)); - pRuntimeEnv->pCtx[i].pOutput = ((char*) pQuery->sdata[i]->data) + pQuery->rec.rows * bytes; - - if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { - pRuntimeEnv->pCtx[i].ptsOutputBuf = pRuntimeEnv->pCtx[0].pOutput; - } +void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size) { + for (int32_t j = 0; j < size; ++j) { + SResultRowCellInfo* pResInfo = GET_RES_INFO(&pCtx[j]); + if (pResInfo->initialized) { + continue; } - updateNumOfResult(pRuntimeEnv, (int32_t)pQuery->rec.rows); + aAggs[pCtx[j].functionId].init(&pCtx[j]); } } -void setQueryStatus(SQuery *pQuery, int8_t status) { +void setQueryStatus(SQueryRuntimeEnv *pRuntimeEnv, int8_t status) { if (status == QUERY_NOT_COMPLETED) { - pQuery->status = status; + pRuntimeEnv->status = status; } else { // QUERY_NOT_COMPLETED is not compatible with any other status, so clear its position first - CLEAR_QUERY_STATUS(pQuery, QUERY_NOT_COMPLETED); - pQuery->status |= status; - } -} - -bool needRepeatScan(SQueryRuntimeEnv *pRuntimeEnv) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - bool toContinue = false; - if (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery)) { - // for each group result, call the finalize function for each column - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; - - for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SResultRow *pResult = getResultRow(pWindowResInfo, i); - - setResultOutputBuf(pRuntimeEnv, pResult); - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int16_t functId = pQuery->pExpr1[j].base.functionId; - if (functId == TSDB_FUNC_TS) { - continue; - } - - aAggs[functId].xNextStep(&pRuntimeEnv->pCtx[j]); - SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); - - toContinue |= (!pResInfo->complete); - } - } - } else { - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int16_t functId = pQuery->pExpr1[j].base.functionId; - if (functId == TSDB_FUNC_TS) { - continue; - } - - aAggs[functId].xNextStep(&pRuntimeEnv->pCtx[j]); - SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); - - toContinue |= (!pResInfo->complete); - } + CLEAR_QUERY_STATUS(pRuntimeEnv, QUERY_NOT_COMPLETED); + pRuntimeEnv->status |= status; } - - return toContinue; -} - -static SQueryStatusInfo getQueryStatusInfo(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { - SQuery *pQuery = pRuntimeEnv->pQuery; - STableQueryInfo* pTableQueryInfo = pQuery->current; - - assert((start <= pTableQueryInfo->lastKey && QUERY_IS_ASC_QUERY(pQuery)) || - (start >= pTableQueryInfo->lastKey && !QUERY_IS_ASC_QUERY(pQuery))); - - SQueryStatusInfo info = { - .status = pQuery->status, - .windowIndex = pRuntimeEnv->resultRowInfo.curIndex, - .lastKey = start, - }; - - TIME_WINDOW_COPY(info.w, pQuery->window); - return info; } -static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusInfo *pStatus) { - SQInfo *pQInfo = GET_QINFO_ADDR(pRuntimeEnv); +static void setupEnvForReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, SQLFunctionCtx* pCtx, int32_t numOfOutput) { SQuery *pQuery = pRuntimeEnv->pQuery; - pStatus->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); // save the cursor if (pRuntimeEnv->pTsBuf) { SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); @@ -3437,228 +3091,49 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI // reverse order time range SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); - SWITCH_ORDER(pQuery->order.order); - - if (QUERY_IS_ASC_QUERY(pQuery)) { - assert(pQuery->window.skey <= pQuery->window.ekey); - } else { - assert(pQuery->window.skey >= pQuery->window.ekey); - } SET_REVERSE_SCAN_FLAG(pRuntimeEnv); - STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); - - setQueryStatus(pQuery, QUERY_NOT_COMPLETED); - switchCtxOrder(pRuntimeEnv); - disableFuncInReverseScan(pQInfo); - setupQueryRangeForReverseScan(pQInfo); - - // clean unused handle - if (pRuntimeEnv->pSecQueryHandle != NULL) { - tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); - } - - pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); - if (pRuntimeEnv->pSecQueryHandle == NULL) { - longjmp(pRuntimeEnv->env, terrno); - } -} - -static void clearEnvAfterReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusInfo *pStatus) { - SQuery *pQuery = pRuntimeEnv->pQuery; - STableQueryInfo* pTableQueryInfo = pQuery->current; + setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); + switchCtxOrder(pCtx, numOfOutput); SWITCH_ORDER(pQuery->order.order); - switchCtxOrder(pRuntimeEnv); - - tsBufSetCursor(pRuntimeEnv->pTsBuf, &pStatus->cur); - if (pRuntimeEnv->pTsBuf) { - pRuntimeEnv->pTsBuf->cur.order = pQuery->order.order; - } - - SET_MASTER_SCAN_FLAG(pRuntimeEnv); - - // update the pQuery->window.skey and pQuery->window.ekey to limit the scan scope of sliding query during reverse scan - pTableQueryInfo->lastKey = pStatus->lastKey; - pQuery->status = pStatus->status; - - pTableQueryInfo->win = pStatus->w; - pQuery->window = pTableQueryInfo->win; -} - -static void restoreTimeWindow(STableGroupInfo* pTableGroupInfo, STsdbQueryCond* pCond) { - assert(pTableGroupInfo->numOfTables == 1); - SArray* pTableKeyGroup = taosArrayGetP(pTableGroupInfo->pGroupList, 0); - STableKeyInfo* pKeyInfo = taosArrayGet(pTableKeyGroup, 0); - pKeyInfo->lastKey = pCond->twindow.skey; -} - -static void handleInterpolationQuery(SQInfo* pQInfo) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - - SQuery *pQuery = pRuntimeEnv->pQuery; - if (pQuery->numOfCheckedBlocks > 0 || !isPointInterpoQuery(pQuery)) { - return; - } - - SArray *prev = tsdbGetExternalRow(pRuntimeEnv->pQueryHandle, &pQInfo->memRef, TSDB_PREV_ROW); - SArray *next = tsdbGetExternalRow(pRuntimeEnv->pQueryHandle, &pQInfo->memRef, TSDB_NEXT_ROW); - if (prev == NULL || next == NULL) { - return; - } - - // setup the pCtx->start/end info and calculate the interpolation value - SColumnInfoData *startTs = taosArrayGet(prev, 0); - SColumnInfoData *endTs = taosArrayGet(next, 0); - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; - - int32_t functionId = pQuery->pExpr1[i].base.functionId; - SColIndex *pColIndex = &pQuery->pExpr1[i].base.colInfo; - - if (!TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { - aAggs[functionId].xFunction(pCtx); - continue; - } - - SColumnInfoData *p = taosArrayGet(prev, pColIndex->colIndex); - SColumnInfoData *n = taosArrayGet(next, pColIndex->colIndex); - - assert(p->info.colId == pColIndex->colId); - - pCtx->start.key = *(TSKEY *)startTs->pData; - pCtx->end.key = *(TSKEY *)endTs->pData; - - if (p->info.type != TSDB_DATA_TYPE_BINARY && p->info.type != TSDB_DATA_TYPE_NCHAR) { - GET_TYPED_DATA(pCtx->start.val, double, p->info.type, p->pData); - GET_TYPED_DATA(pCtx->end.val, double, n->info.type, n->pData); - } else { // string pointer - pCtx->start.ptr = p->pData; - pCtx->end.ptr = n->pData; - } - - pCtx->param[2].i64 = (int8_t)pQuery->fillType; - pCtx->startTs = pQuery->window.skey; - if (pQuery->fillVal != NULL) { - if (isNull((const char *)&pQuery->fillVal[i], pCtx->inputType)) { - pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; - } else { // todo refactor, tVariantCreateFromBinary should handle the NULL value - if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) { - tVariantCreateFromBinary(&pCtx->param[1], (char *)&pQuery->fillVal[i], pCtx->inputBytes, pCtx->inputType); - } - } - } - - aAggs[functionId].xFunction(pCtx); - } -} - -void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { - SQInfo *pQInfo = (SQInfo *) GET_QINFO_ADDR(pRuntimeEnv); - SQuery *pQuery = pRuntimeEnv->pQuery; - STableQueryInfo *pTableQueryInfo = pQuery->current; - - setQueryStatus(pQuery, QUERY_NOT_COMPLETED); - - // store the start query position - SQueryStatusInfo qstatus = getQueryStatusInfo(pRuntimeEnv, start); - SET_MASTER_SCAN_FLAG(pRuntimeEnv); - - if (!pRuntimeEnv->groupbyColumn && pRuntimeEnv->hasTagResults) { - setTagVal(pRuntimeEnv, pTableQueryInfo->pTable); - } - - while (1) { - doScanAllDataBlocks(pRuntimeEnv); - - if (pRuntimeEnv->scanFlag == MASTER_SCAN) { - qstatus.status = pQuery->status; - - // do nothing if no data blocks are found qualified during scan - if (qstatus.lastKey == pTableQueryInfo->lastKey) { - qDebug("QInfo:%p no results generated in this scan", pQInfo); - } - } - - if (!needRepeatScan(pRuntimeEnv)) { - // restore the status code and jump out of loop - if (pRuntimeEnv->scanFlag == REPEAT_SCAN) { - pQuery->status = qstatus.status; - } - - break; - } - - if (pRuntimeEnv->pSecQueryHandle != NULL) { - tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); - } - - STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); - restoreTimeWindow(&pQInfo->tableGroupInfo, &cond); - pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); - if (pRuntimeEnv->pSecQueryHandle == NULL) { - longjmp(pRuntimeEnv->env, terrno); - } - - pRuntimeEnv->resultRowInfo.curIndex = qstatus.windowIndex; - setQueryStatus(pQuery, QUERY_NOT_COMPLETED); - pRuntimeEnv->scanFlag = REPEAT_SCAN; - - if (pRuntimeEnv->pTsBuf) { - bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); - assert(ret); - } - - qDebug("QInfo:%p start to repeat scan data blocks due to query func required, qrange:%"PRId64"-%"PRId64, pQInfo, - cond.twindow.skey, cond.twindow.ekey); - } - - if (needReverseScan(pQuery)) { - setEnvBeforeReverseScan(pRuntimeEnv, &qstatus); - - // reverse scan from current position - qDebug("QInfo:%p start to reverse scan", pQInfo); - doScanAllDataBlocks(pRuntimeEnv); - - clearEnvAfterReverseScan(pRuntimeEnv, &qstatus); - } - handleInterpolationQuery(pQInfo); + setupQueryRangeForReverseScan(pRuntimeEnv); } -void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { +void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { + SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; SQuery *pQuery = pRuntimeEnv->pQuery; - if (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery)) { + int32_t numOfOutput = pOperator->numOfOutput; + if (pQuery->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery) || pQuery->sw.gap > 0) { // for each group result, call the finalize function for each column - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; - if (pRuntimeEnv->groupbyColumn) { - closeAllResultRows(pWindowResInfo); + if (pQuery->groupbyColumn) { + closeAllResultRows(pResultRowInfo); } - for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SResultRow *buf = pWindowResInfo->pResult[i]; - if (!isResultRowClosed(pWindowResInfo, i)) { + for (int32_t i = 0; i < pResultRowInfo->size; ++i) { + SResultRow *buf = pResultRowInfo->pResult[i]; + if (!isResultRowClosed(pResultRowInfo, i)) { continue; } - setResultOutputBuf(pRuntimeEnv, buf); + setResultOutputBuf(pRuntimeEnv, buf, pCtx, numOfOutput, rowCellInfoOffset); - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - aAggs[pQuery->pExpr1[j].base.functionId].xFinalize(&pRuntimeEnv->pCtx[j]); + for (int32_t j = 0; j < numOfOutput; ++j) { + aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]); } /* * set the number of output results for group by normal columns, the number of output rows usually is 1 except * the top and bottom query */ - buf->numOfRows = (uint16_t)getNumOfResult(pRuntimeEnv); + buf->numOfRows = (uint16_t)getNumOfResult(pRuntimeEnv, pCtx, numOfOutput); } } else { - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - aAggs[pQuery->pExpr1[j].base.functionId].xFinalize(&pRuntimeEnv->pCtx[j]); + for (int32_t j = 0; j < numOfOutput; ++j) { + aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]); } } } @@ -3706,25 +3181,40 @@ void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) { cleanupResultRowInfo(&pTableQueryInfo->resInfo); } -/** - * set output buffer for different group - * @param pRuntimeEnv - * @param pDataBlockInfo - */ -void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - STableQueryInfo *pTableQueryInfo = pRuntimeEnv->pQuery->current; - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; +void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, + int32_t numOfOutput, int32_t* rowCellInfoOffset) { + // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group + tFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); - // lastKey needs to be updated - pTableQueryInfo->lastKey = nextKey; - if (pRuntimeEnv->prevGroupId != INT32_MIN && pRuntimeEnv->prevGroupId == groupIndex) { - return; + int16_t offset = 0; + for (int32_t i = 0; i < numOfOutput; ++i) { + pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset); + + SResultRowCellInfo* pResInfo = pCtx[i].resultInfo; + if (pResInfo->initialized && pResInfo->complete) { + offset += pCtx[i].outputBytes; + continue; + } + + pCtx[i].pOutput = getPosInResultPage(pRuntimeEnv->pQuery, bufPage, pResult->offset, offset); + offset += pCtx[i].outputBytes; + + int32_t functionId = pCtx[i].functionId; + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { + pCtx[i].ptsOutputBuf = pCtx[0].pOutput; + } + + if (!pResInfo->initialized) { + aAggs[functionId].init(&pCtx[i]); + } } +} +void doSetTableGroupOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, SQLFunctionCtx* pCtx, + int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t groupIndex) { int64_t uid = 0; - SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex, - sizeof(groupIndex), true, uid); + SResultRow* pResultRow = + doPrepareResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char*)&groupIndex, sizeof(groupIndex), true, uid); assert (pResultRow != NULL); /* @@ -3732,142 +3222,146 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { * all group belong to one result set, and each group result has different group id so set the id to be one */ if (pResultRow->pageId == -1) { - if (addNewWindowResultBuf(pResultRow, pRuntimeEnv->pResultBuf, groupIndex, pRuntimeEnv->numOfRowsPerPage) != - TSDB_CODE_SUCCESS) { + int32_t ret = addNewWindowResultBuf(pResultRow, pRuntimeEnv->pResultBuf, groupIndex, pRuntimeEnv->pQuery->resultRowSize); + if (ret != TSDB_CODE_SUCCESS) { return; } } + setResultRowOutputBufInitCtx(pRuntimeEnv, pResultRow, pCtx, numOfOutput, rowCellInfoOffset); +} + +void setExecutionContext(SQueryRuntimeEnv* pRuntimeEnv, SOptrBasicInfo* pInfo, int32_t numOfOutput, int32_t groupIndex, + TSKEY nextKey) { + STableQueryInfo *pTableQueryInfo = pRuntimeEnv->pQuery->current; + + // lastKey needs to be updated + pTableQueryInfo->lastKey = nextKey; + if (pRuntimeEnv->prevGroupId != INT32_MIN && pRuntimeEnv->prevGroupId == groupIndex) { + return; + } + + doSetTableGroupOutputBuf(pRuntimeEnv, &pInfo->resultRowInfo, pInfo->pCtx, pInfo->rowCellInfoOffset, numOfOutput, groupIndex); + // record the current active group id pRuntimeEnv->prevGroupId = groupIndex; - setResultOutputBuf(pRuntimeEnv, pResultRow); - initCtxOutputBuf(pRuntimeEnv); } -void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult) { - SQuery *pQuery = pRuntimeEnv->pQuery; - +void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, + int32_t numOfCols, int32_t* rowCellInfoOffset) { // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; - pCtx->pOutput = getPosInResultPage(pRuntimeEnv, i, pResult, page); + int16_t offset = 0; + for (int32_t i = 0; i < numOfCols; ++i) { + pCtx[i].pOutput = getPosInResultPage(pRuntimeEnv->pQuery, page, pResult->offset, offset); + offset += pCtx[i].outputBytes; - int32_t functionId = pQuery->pExpr1[i].base.functionId; + int32_t functionId = pCtx[i].functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].pOutput; + pCtx[i].ptsOutputBuf = pCtx[0].pOutput; } /* * set the output buffer information and intermediate buffer, * not all queries require the interResultBuf, such as COUNT */ - pCtx->resultInfo = getResultCell(pRuntimeEnv, pResult, i); + pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset); } } -void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group - tFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); +void setCtxTagForJoin(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable) { + SQuery* pQuery = pRuntimeEnv->pQuery; - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; + SSqlFuncMsg* pFuncMsg = &pExprInfo->base; + if (pQuery->stableQuery && (pRuntimeEnv->pTsBuf != NULL) && + (pFuncMsg->functionId == TSDB_FUNC_TS || pFuncMsg->functionId == TSDB_FUNC_PRJ) && + (pFuncMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX)) { + assert(pFuncMsg->numOfParams == 1); - pCtx->resultInfo = getResultCell(pRuntimeEnv, pResult, i); - if (pCtx->resultInfo->initialized && pCtx->resultInfo->complete) { - continue; - } + int16_t tagColId = (int16_t)pExprInfo->base.arg->argValue.i64; + SColumnInfo* pColInfo = doGetTagColumnInfoById(pQuery->tagColList, pQuery->numOfTags, tagColId); - pCtx->pOutput = getPosInResultPage(pRuntimeEnv, i, pResult, bufPage); - pCtx->currentStage = 0; + doSetTagValueInParam(pTable, tagColId, &pCtx->tag, pColInfo->type, pColInfo->bytes); - int32_t functionId = pCtx->functionId; - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].pOutput; - } - - if (!pCtx->resultInfo->initialized) { - aAggs[functionId].init(pCtx); + int16_t tagType = pCtx[0].tag.nType; + if (tagType == TSDB_DATA_TYPE_BINARY || tagType == TSDB_DATA_TYPE_NCHAR) { + qDebug("QInfo:%p set tag value for join comparison, colId:%" PRId64 ", val:%s", pRuntimeEnv->qinfo, + pExprInfo->base.arg->argValue.i64, pCtx[0].tag.pz); + } else { + qDebug("QInfo:%p set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64, pRuntimeEnv->qinfo, + pExprInfo->base.arg->argValue.i64, pCtx[0].tag.i64); } } } -int32_t setTimestampListJoinInfo(SQInfo *pQInfo, STableQueryInfo *pTableQueryInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; +int32_t setTimestampListJoinInfo(SQueryRuntimeEnv* pRuntimeEnv, tVariant* pTag, STableQueryInfo *pTableQueryInfo) { + SQuery* pQuery = pRuntimeEnv->pQuery; + assert(pRuntimeEnv->pTsBuf != NULL); // both the master and supplement scan needs to set the correct ts comp start position - tVariant* pTag = &pRuntimeEnv->pCtx[0].tag; - if (pTableQueryInfo->cur.vgroupIndex == -1) { tVariantAssign(&pTableQueryInfo->tag, pTag); - STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTsBuf, pQInfo->vgId, &pTableQueryInfo->tag); + STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTsBuf, pQuery->vgId, &pTableQueryInfo->tag); // failed to find data with the specified tag value and vnodeId if (!tsBufIsValidElem(&elem)) { if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz); + qError("QInfo:%p failed to find tag:%s in ts_comp", pRuntimeEnv->qinfo, pTag->pz); } else { - qError("QInfo:%p failed to find tag:%" PRId64 " in ts_comp", pQInfo, pTag->i64); + qError("QInfo:%p failed to find tag:%" PRId64 " in ts_comp", pRuntimeEnv->qinfo, pTag->i64); } - return false; + return -1; } - // keep the cursor info of current meter + // Keep the cursor info of current table pTableQueryInfo->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pRuntimeEnv->qinfo, pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } else { - qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pRuntimeEnv->qinfo, pTag->i64, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } } else { tsBufSetCursor(pRuntimeEnv->pTsBuf, &pTableQueryInfo->cur); - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pRuntimeEnv->qinfo, pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } else { - qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pRuntimeEnv->qinfo, pTag->i64, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } } return 0; } -int32_t setParamValue(SQueryRuntimeEnv* pRuntimeEnv) { +void setParamForStableStddev(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr) { SQuery* pQuery = pRuntimeEnv->pQuery; - if (pRuntimeEnv->prevResult == NULL || pRuntimeEnv->groupbyColumn) { - return TSDB_CODE_SUCCESS; - } - int32_t numOfExprs = pQuery->numOfOutput; for(int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExprInfo = &(pQuery->pExpr1[i]); - if(pExprInfo->base.functionId != TSDB_FUNC_STDDEV_DST) { + SExprInfo* pExprInfo = &(pExpr[i]); + if (pExprInfo->base.functionId != TSDB_FUNC_STDDEV_DST) { continue; } SSqlFuncMsg* pFuncMsg = &pExprInfo->base; - pRuntimeEnv->pCtx[i].param[0].arr = NULL; - pRuntimeEnv->pCtx[i].param[0].nType = TSDB_DATA_TYPE_INT; // avoid freeing the memory by setting the type to be int + pCtx[i].param[0].arr = NULL; + pCtx[i].param[0].nType = TSDB_DATA_TYPE_INT; // avoid freeing the memory by setting the type to be int - int32_t numOfGroup = (int32_t) taosArrayGetSize(pRuntimeEnv->prevResult); - for(int32_t j = 0; j < numOfGroup; ++j) { - SInterResult *p = taosArrayGet(pRuntimeEnv->prevResult, j); + // TODO use hash to speedup this loop + int32_t numOfGroup = (int32_t)taosArrayGetSize(pRuntimeEnv->prevResult); + for (int32_t j = 0; j < numOfGroup; ++j) { + SInterResult* p = taosArrayGet(pRuntimeEnv->prevResult, j); if (pQuery->tagLen == 0 || memcmp(p->tags, pRuntimeEnv->tagVal, pQuery->tagLen) == 0) { - - int32_t numOfCols = (int32_t) taosArrayGetSize(p->pResult); - for(int32_t k = 0; k < numOfCols; ++k) { + int32_t numOfCols = (int32_t)taosArrayGetSize(p->pResult); + for (int32_t k = 0; k < numOfCols; ++k) { SStddevInterResult* pres = taosArrayGet(p->pResult, k); if (pres->colId == pFuncMsg->colInfo.colId) { - pRuntimeEnv->pCtx[i].param[0].arr = pres->pResult; + pCtx[i].param[0].arr = pres->pResult; break; } } @@ -3875,7 +3369,6 @@ int32_t setParamValue(SQueryRuntimeEnv* pRuntimeEnv) { } } - return 0; } /* @@ -3887,84 +3380,61 @@ int32_t setParamValue(SQueryRuntimeEnv* pRuntimeEnv) { * merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there * is a previous result generated or not. */ -void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; - STableQueryInfo *pTableQueryInfo = pQuery->current; - - if (pTableQueryInfo->queryRangeSet) { - pTableQueryInfo->lastKey = key; - } else { - pTableQueryInfo->win.skey = key; - STimeWindow win = {.skey = key, .ekey = pQuery->window.ekey}; +void setIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, TSKEY key) { + SQuery *pQuery = pRuntimeEnv->pQuery; + STableQueryInfo *pTableQueryInfo = pQuery->current; + SResultRowInfo *pWindowResInfo = &pTableQueryInfo->resInfo; - // for too small query range, no data in this interval. - if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.ekey < pQuery->window.skey)) || - (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.skey < pQuery->window.ekey))) { - return; - } + if (pWindowResInfo->prevSKey != TSKEY_INITIAL_VAL) { + return; + } - /** - * In handling the both ascending and descending order super table query, we need to find the first qualified - * timestamp of this table, and then set the first qualified start timestamp. - * In ascending query, the key is the first qualified timestamp. However, in the descending order query, additional - * operations involve. - */ - STimeWindow w = TSWINDOW_INITIALIZER; - SResultRowInfo *pWindowResInfo = &pTableQueryInfo->resInfo; + pTableQueryInfo->win.skey = key; + STimeWindow win = {.skey = key, .ekey = pQuery->window.ekey}; - TSKEY sk = MIN(win.skey, win.ekey); - TSKEY ek = MAX(win.skey, win.ekey); - getAlignQueryTimeWindow(pQuery, win.skey, sk, ek, &w); + /** + * In handling the both ascending and descending order super table query, we need to find the first qualified + * timestamp of this table, and then set the first qualified start timestamp. + * In ascending query, the key is the first qualified timestamp. However, in the descending order query, additional + * operations involve. + */ + STimeWindow w = TSWINDOW_INITIALIZER; - if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { - if (!QUERY_IS_ASC_QUERY(pQuery)) { - assert(win.ekey == pQuery->window.ekey); - } + TSKEY sk = MIN(win.skey, win.ekey); + TSKEY ek = MAX(win.skey, win.ekey); + getAlignQueryTimeWindow(pQuery, win.skey, sk, ek, &w); - pWindowResInfo->prevSKey = w.skey; + if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { + if (!QUERY_IS_ASC_QUERY(pQuery)) { + assert(win.ekey == pQuery->window.ekey); } - pTableQueryInfo->queryRangeSet = 1; - pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; + pWindowResInfo->prevSKey = w.skey; } -} -bool requireTimestamp(SQuery *pQuery) { - for (int32_t i = 0; i < pQuery->numOfOutput; i++) { - int32_t functionId = pQuery->pExpr1[i].base.functionId; - if ((aAggs[functionId].status & TSDB_FUNCSTATE_NEED_TS) != 0) { - return true; - } - } - return false; + pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; } -bool needPrimaryTimestampCol(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo) { - /* - * 1. if skey or ekey locates in this block, we need to load the timestamp column to decide the precise position - * 2. if there are top/bottom, first_dst/last_dst functions, we need to load timestamp column in any cases; - */ - STimeWindow *w = &pDataBlockInfo->window; - STableQueryInfo* pTableQueryInfo = pQuery->current; - - bool loadPrimaryTS = (pTableQueryInfo->lastKey >= w->skey && pTableQueryInfo->lastKey <= w->ekey) || - (pQuery->window.ekey >= w->skey && pQuery->window.ekey <= w->ekey) || requireTimestamp(pQuery); - - return loadPrimaryTS; -} +/** + * copyToOutputBuf support copy data in ascending/descending order + * For interval query of both super table and table, copy the data in ascending order, since the output results are + * ordered in SWindowResutl already. While handling the group by query for both table and super table, + * all group result are completed already. + * + * @param pQInfo + * @param result + */ -static int32_t doCopyToSData(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType) { - void* qinfo = GET_QINFO_ADDR(pRuntimeEnv); +static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock) { SQuery *pQuery = pRuntimeEnv->pQuery; int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); - int32_t numOfResult = (int32_t) pQuery->rec.rows; // there are already exists result rows + int32_t numOfResult = pBlock->info.rows; // there are already exists result rows int32_t start = 0; int32_t step = -1; - qDebug("QInfo:%p start to copy data from windowResInfo to output buf", qinfo); + qDebug("QInfo:%p start to copy data from windowResInfo to output buf", pRuntimeEnv->qinfo); if (orderType == TSDB_ORDER_ASC) { start = pGroupResInfo->index; step = 1; @@ -3982,56 +3452,56 @@ static int32_t doCopyToSData(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGrou int32_t numOfRowsToCopy = pRow->numOfRows; - //current output space is not enough to accommodate all data of this page, prepare more space - if (numOfRowsToCopy > (pQuery->rec.capacity - numOfResult)) { - int32_t newSize = (int32_t) (pQuery->rec.capacity + (numOfRowsToCopy - numOfResult)); - expandBuffer(pRuntimeEnv, newSize, GET_QINFO_ADDR(pRuntimeEnv)); - } - pGroupResInfo->index += 1; tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pRow->pageId); - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int32_t size = pRuntimeEnv->pCtx[j].outputBytes; - char *out = pQuery->sdata[j]->data + numOfResult * size; - char *in = getPosInResultPage(pRuntimeEnv, j, pRow, page); - memcpy(out, in, size * numOfRowsToCopy); + int16_t offset = 0; + for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, j); + int32_t bytes = pColInfoData->info.bytes; + + char *out = pColInfoData->pData + numOfResult * bytes; + char *in = getPosInResultPage(pQuery, page, pRow->offset, offset); + memcpy(out, in, bytes * numOfRowsToCopy); + + offset += bytes; } numOfResult += numOfRowsToCopy; - if (numOfResult == pQuery->rec.capacity) { // output buffer is full + if (numOfResult == pRuntimeEnv->resultInfo.capacity) { // output buffer is full break; } } - qDebug("QInfo:%p copy data to query buf completed", qinfo); - return numOfResult; + qDebug("QInfo:%p copy data to query buf completed", pRuntimeEnv->qinfo); + pBlock->info.rows = numOfResult; + return 0; } -/** - * copyToOutputBuf support copy data in ascending/descending order - * For interval query of both super table and table, copy the data in ascending order, since the output results are - * ordered in SWindowResutl already. While handling the group by query for both table and super table, - * all group result are completed already. - * - * @param pQInfo - * @param result - */ -void copyToOutputBuf(SQInfo *pQInfo, SResultRowInfo *pResultInfo) { - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - SGroupResInfo *pGroupResInfo = &pQInfo->groupResInfo; +static void toSSDataBlock(SGroupResInfo *pGroupResInfo, SQueryRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock) { + assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup); - assert(pQuery->rec.rows == 0 && pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup); - if (!hasRemainData(pGroupResInfo)) { + pBlock->info.rows = 0; + if (!hasRemainDataInCurrentGroup(pGroupResInfo)) { return; } + SQuery* pQuery = pRuntimeEnv->pQuery; int32_t orderType = (pQuery->pGroupbyExpr != NULL) ? pQuery->pGroupbyExpr->orderType : TSDB_ORDER_ASC; - pQuery->rec.rows = doCopyToSData(&pQInfo->runtimeEnv, pGroupResInfo, orderType); -} + doCopyToSDataBlock(pRuntimeEnv, pGroupResInfo, orderType, pBlock); -static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv) { + SColumnInfoData* pInfoData = taosArrayGet(pBlock->pDataBlock, 0); + + if (pInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP) { + STimeWindow* w = &pBlock->info.window; + w->skey = *(int64_t*)pInfoData->pData; + w->ekey = *(int64_t*)(((char*)pInfoData->pData) + TSDB_KEYSIZE * (pBlock->info.rows - 1)); + } +} + +static void updateNumOfRowsInResultRows(SQueryRuntimeEnv *pRuntimeEnv, + SQLFunctionCtx* pCtx, int32_t numOfOutput, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { SQuery *pQuery = pRuntimeEnv->pQuery; // update the number of result for each, only update the number of rows for the corresponding window result. @@ -4039,108 +3509,47 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv) { return; } - for (int32_t i = 0; i < pRuntimeEnv->resultRowInfo.size; ++i) { - SResultRow *pResult = pRuntimeEnv->resultRowInfo.pResult[i]; + for (int32_t i = 0; i < pResultRowInfo->size; ++i) { + SResultRow *pResult = pResultRowInfo->pResult[i]; - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int32_t functionId = pRuntimeEnv->pCtx[j].functionId; + for (int32_t j = 0; j < numOfOutput; ++j) { + int32_t functionId = pCtx[j].functionId; if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ) { continue; } - SResultRowCellInfo* pCell = getResultCell(pRuntimeEnv, pResult, j); + SResultRowCellInfo* pCell = getResultCell(pResult, j, rowCellInfoOffset); pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pCell->numOfRes)); } } } -static void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, - SArray *pDataBlock, __block_search_fn_t searchFn) { - SQuery * pQuery = pRuntimeEnv->pQuery; - STableQueryInfo* pTableQueryInfo = pQuery->current; - - SResultRowInfo * pResultRowInfo = &pTableQueryInfo->resInfo; - pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : pDataBlockInfo->rows - 1; - - if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->groupbyColumn) { - rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, pDataBlock); - } else { - blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, searchFn, pDataBlock); - } - - if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - updateResultRowIndex(pResultRowInfo, pTableQueryInfo, QUERY_IS_ASC_QUERY(pQuery), pRuntimeEnv->timeWindowInterpo); - } -} - -bool hasNotReturnedResults(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo) { +static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pRuntimeEnv->pQuery; - SFillInfo *pFillInfo = pRuntimeEnv->pFillInfo; - - if (!Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - return false; - } - - if (pQuery->limit.limit > 0 && pQuery->rec.total >= pQuery->limit.limit) { - return false; - } - - if (pQuery->fillType != TSDB_FILL_NONE && !isPointInterpoQuery(pQuery)) { - // There are results not returned to client yet, so filling applied to the remain result is required firstly. - if (taosFillHasMoreResults(pFillInfo)) { - return true; - } - - /* - * While the code reaches here, there are no results remains now. - * If query is not completed yet, the gaps between two results blocks need to be handled after next data block - * is retrieved from TSDB. - * - * NOTE: If the result set is not the first block, the gap in front of the result set will be filled. If the result - * set is the FIRST result block, the gap between the start time of query time window and the timestamp of the - * first result row in the actual result set will fill nothing. - */ - int32_t numOfTotal = (int32_t)getNumOfResultsAfterFillGap(pFillInfo, pQuery->window.ekey, (int32_t)pQuery->rec.capacity); - return numOfTotal > 0; - } else { // there are results waiting for returned to client. - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED) && hasRemainData(pGroupResInfo) && - (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery))) { - return true; - } - } - - return false; -} - -static int16_t getNumOfFinalResCol(SQuery* pQuery) { - return pQuery->pExpr2 == NULL? pQuery->numOfOutput:pQuery->numOfExpr2; -} -static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data) { - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + SSDataBlock* pRes = pRuntimeEnv->outputBuf; if (pQuery->pExpr2 == NULL) { for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { - int32_t bytes = pQuery->pExpr1[col].bytes; - - memmove(data, pQuery->sdata[col]->data, bytes * numOfRows); - data += bytes * numOfRows; + SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, col); + memmove(data, pColRes->pData, pColRes->info.bytes * pRes->info.rows); + data += pColRes->info.bytes * pRes->info.rows; } } else { for (int32_t col = 0; col < pQuery->numOfExpr2; ++col) { - int32_t bytes = pQuery->pExpr2[col].bytes; - - memmove(data, pQuery->sdata[col]->data, bytes * numOfRows); - data += bytes * numOfRows; + SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, col); + memmove(data, pColRes->pData, pColRes->info.bytes * numOfRows); + data += pColRes->info.bytes * numOfRows; } } - int32_t numOfTables = (int32_t) taosHashGetSize(pQInfo->arrTableIdInfo); + int32_t numOfTables = (int32_t) taosHashGetSize(pRuntimeEnv->pTableRetrieveTsMap); *(int32_t*)data = htonl(numOfTables); data += sizeof(int32_t); int32_t total = 0; - STableIdInfo* item = taosHashIterate(pQInfo->arrTableIdInfo, NULL); + STableIdInfo* item = taosHashIterate(pRuntimeEnv->pTableRetrieveTsMap, NULL); while(item) { STableIdInfo* pDst = (STableIdInfo*)data; @@ -4152,75 +3561,34 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data total++; qDebug("QInfo:%p set subscribe info, tid:%d, uid:%"PRIu64", skey:%"PRId64, pQInfo, item->tid, item->uid, item->key); - item = taosHashIterate(pQInfo->arrTableIdInfo, item); + item = taosHashIterate(pRuntimeEnv->pTableRetrieveTsMap, item); } qDebug("QInfo:%p set %d subscribe info", pQInfo, total); - // Check if query is completed or not for stable query or normal table query respectively. - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - if (pQInfo->runtimeEnv.stableQuery) { - if (IS_STASBLE_QUERY_OVER(pQInfo)) { - setQueryStatus(pQuery, QUERY_OVER); - } - } else { - if (!hasNotReturnedResults(&pQInfo->runtimeEnv, &pQInfo->groupResInfo)) { - setQueryStatus(pQuery, QUERY_OVER); - } - } + if (Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED) && pRuntimeEnv->proot->status == OP_EXEC_DONE) { + setQueryStatus(pRuntimeEnv, QUERY_OVER); } } -int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst) { - SQInfo *pQInfo = GET_QINFO_ADDR(pRuntimeEnv); - SQuery *pQuery = pRuntimeEnv->pQuery; - SFillInfo* pFillInfo = pRuntimeEnv->pFillInfo; - - while (1) { - int32_t ret = (int32_t)taosFillResultDataBlock(pFillInfo, (tFilePage**)pQuery->sdata, (int32_t)pQuery->rec.capacity); - - // todo apply limit output function - /* reached the start position of according to offset value, return immediately */ - if (pQuery->limit.offset == 0) { - qDebug("QInfo:%p initial numOfRows:%d, generate filled result:%d rows", pQInfo, pFillInfo->numOfRows, ret); - return ret; - } - - if (pQuery->limit.offset < ret) { - qDebug("QInfo:%p initial numOfRows:%d, generate filled result:%d rows, offset:%" PRId64 ". Discard due to offset, remain:%" PRId64 ", new offset:%d", - pQInfo, pFillInfo->numOfRows, ret, pQuery->limit.offset, ret - pQuery->limit.offset, 0); - - ret -= (int32_t)pQuery->limit.offset; - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { //???pExpr1 or pExpr2 - memmove(pDst[i]->data, pDst[i]->data + pQuery->pExpr1[i].bytes * pQuery->limit.offset, - ret * pQuery->pExpr1[i].bytes); - } - - pQuery->limit.offset = 0; - return ret; - } else { - qDebug("QInfo:%p initial numOfRows:%d, generate filled result:%d rows, offset:%" PRId64 ". Discard due to offset, " - "remain:%d, new offset:%" PRId64, pQInfo, pFillInfo->numOfRows, ret, pQuery->limit.offset, 0, - pQuery->limit.offset - ret); - - pQuery->limit.offset -= ret; - ret = 0; - } - - // no data in current data after fill - int32_t numOfTotal = (int32_t)getNumOfResultsAfterFillGap(pFillInfo, pFillInfo->end, (int32_t)pQuery->rec.capacity); - if (numOfTotal == 0) { - return 0; - } +int32_t doFillTimeIntervalGapsInResults(SFillInfo* pFillInfo, SSDataBlock *pOutput, int32_t capacity) { + void** p = calloc(pFillInfo->numOfCols, POINTER_BYTES); + for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pOutput->pDataBlock, i); + p[i] = pColInfoData->pData; } + + pOutput->info.rows = (int32_t)taosFillResultDataBlock(pFillInfo, p, capacity); + tfree(p); + return pOutput->info.rows; } void queryCostStatis(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQueryCostInfo *pSummary = &pRuntimeEnv->summary; + SQueryCostInfo *pSummary = &pQInfo->summary; uint64_t hashSize = taosHashGetMemSize(pQInfo->runtimeEnv.pResultRowHashTable); - hashSize += taosHashGetMemSize(pQInfo->tableqinfoGroupInfo.map); + hashSize += taosHashGetMemSize(pRuntimeEnv->tableqinfoGroupInfo.map); pSummary->hashSize = hashSize; // add the merge time @@ -4244,293 +3612,241 @@ void queryCostStatis(SQInfo *pQInfo) { pSummary->numOfTimeWindows, pSummary->tableInfoSize/1024.0, pSummary->hashSize/1024.0); } -static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { - SQuery *pQuery = pRuntimeEnv->pQuery; - STableQueryInfo* pTableQueryInfo = pQuery->current; - - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - - if (pQuery->limit.offset == pBlockInfo->rows) { // current block will ignore completed - pTableQueryInfo->lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pBlockInfo->window.ekey + step : pBlockInfo->window.skey + step; - pQuery->limit.offset = 0; - return; - } - - if (QUERY_IS_ASC_QUERY(pQuery)) { - pQuery->pos = (int32_t)pQuery->limit.offset; - } else { - pQuery->pos = pBlockInfo->rows - (int32_t)pQuery->limit.offset - 1; - } - - assert(pQuery->pos >= 0 && pQuery->pos <= pBlockInfo->rows - 1); - - SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); - SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); - - // update the pQuery->limit.offset value, and pQuery->pos value - TSKEY *keys = (TSKEY *) pColInfoData->pData; - - // update the offset value - pTableQueryInfo->lastKey = keys[pQuery->pos]; - pQuery->limit.offset = 0; - - int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); - - qDebug("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%"PRId64, GET_QINFO_ADDR(pRuntimeEnv), - pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, pQuery->current->lastKey); -} -static void freeTableBlockDist(STableBlockDist *pTableBlockDist) { - if (pTableBlockDist != NULL) { - taosArrayDestroy(pTableBlockDist->dataBlockInfos); - free(pTableBlockDist->result); - free(pTableBlockDist); - } -} -static int32_t getPercentileFromSortedArray(const SArray* pArray, double rate) { - int32_t len = (int32_t)taosArrayGetSize(pArray); - if (len <= 0) { - return 0; - } - assert(rate >= 0 && rate <= 1.0); - int idx = (int32_t)((len - 1) * rate); - return ((SDataBlockInfo *)(taosArrayGet(pArray, idx)))->rows; -} -static int compareBlockInfo(const void *pLeft, const void *pRight) { - int32_t left = ((SDataBlockInfo *)pLeft)->rows; - int32_t right = ((SDataBlockInfo *)pRight)->rows; - if (left > right) return 1; - if (left < right) return -1; - return 0; -} - -static void generateBlockDistResult(STableBlockDist *pTableBlockDist) { - if (pTableBlockDist == NULL) { - return; - } - int64_t min = 0, max = 0, avg = 0; - SArray* blockInfos= pTableBlockDist->dataBlockInfos; - int64_t totalRows = 0, totalBlocks = taosArrayGetSize(blockInfos); - for (size_t i = 0; i < taosArrayGetSize(blockInfos); i++) { - SDataBlockInfo *blockInfo = taosArrayGet(blockInfos, i); - int64_t rows = blockInfo->rows; - min = MIN(min, rows); - max = MAX(max, rows); - totalRows += rows; - } - avg = totalBlocks > 0 ? (int64_t)(totalRows/totalBlocks) : 0; - - taosArraySort(blockInfos, compareBlockInfo); - - int sz = sprintf(pTableBlockDist->result, - "summery: \n\t 5th=[%d], 25th=[%d], 50th=[%d],75th=[%d], 95th=[%d], 99th=[%d] \n\t min=[%"PRId64"], max=[%"PRId64"], avg = [%"PRId64"] \n\t totalRows=[%"PRId64"], totalBlocks=[%"PRId64"] \n\t seekHeaderTimeCost=[%"PRId64"(us)] \n\t rowsInMem=[%"PRId64"]", - getPercentileFromSortedArray(blockInfos, 0.05), getPercentileFromSortedArray(blockInfos, 0.25), getPercentileFromSortedArray(blockInfos, 0.50), - getPercentileFromSortedArray(blockInfos, 0.75), getPercentileFromSortedArray(blockInfos, 0.95), getPercentileFromSortedArray(blockInfos, 0.99), - min, max, avg, - totalRows, totalBlocks, - pTableBlockDist->firstSeekTimeUs, - pTableBlockDist->numOfRowsInMemTable); - UNUSED(sz); - return; -} -void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - if (pQuery->limit.offset <= 0 || pQuery->numOfFilterCols > 0) { - return; - } - - pQuery->pos = 0; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - - STableQueryInfo* pTableQueryInfo = pQuery->current; - TsdbQueryHandleT pQueryHandle = pRuntimeEnv->pQueryHandle; - - SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; - while (tsdbNextDataBlock(pQueryHandle)) { - if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { - longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); - } - - tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); - - if (pQuery->limit.offset > blockInfo.rows) { - pQuery->limit.offset -= blockInfo.rows; - pTableQueryInfo->lastKey = (QUERY_IS_ASC_QUERY(pQuery)) ? blockInfo.window.ekey : blockInfo.window.skey; - pTableQueryInfo->lastKey += step; - - qDebug("QInfo:%p skip rows:%d, offset:%" PRId64, GET_QINFO_ADDR(pRuntimeEnv), blockInfo.rows, - pQuery->limit.offset); - } else { // find the appropriated start position in current block - updateOffsetVal(pRuntimeEnv, &blockInfo); - break; - } - } - - if (terrno != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, terrno); - } -} - -static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* win, SDataBlockInfo* pBlockInfo, STableQueryInfo* pTableQueryInfo) { - SQuery *pQuery = pRuntimeEnv->pQuery; - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; - - assert(pQuery->limit.offset == 0); - STimeWindow tw = *win; - getNextTimeWindow(pQuery, &tw); - - if ((tw.skey <= pBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (tw.ekey >= pBlockInfo->window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { - - // load the data block and check data remaining in current data block - // TODO optimize performance - SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); - SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); - - tw = *win; - int32_t startPos = - getNextQualifiedWindow(pRuntimeEnv, &tw, pBlockInfo, pColInfoData->pData, binarySearchForKey, -1); - assert(startPos >= 0); - - // set the abort info - pQuery->pos = startPos; - - // reset the query start timestamp - pTableQueryInfo->win.skey = ((TSKEY *)pColInfoData->pData)[startPos]; - pQuery->window.skey = pTableQueryInfo->win.skey; - TSKEY key = pTableQueryInfo->win.skey; - - pWindowResInfo->prevSKey = tw.skey; - int32_t index = pRuntimeEnv->resultRowInfo.curIndex; - - int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); - pRuntimeEnv->resultRowInfo.curIndex = index; // restore the window index - - qDebug("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%" PRId64, - GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, - pQuery->current->lastKey); - - return key; - } else { // do nothing - pQuery->window.skey = tw.skey; - pWindowResInfo->prevSKey = tw.skey; - pTableQueryInfo->lastKey = tw.skey; - - return tw.skey; - } - - return true; -} - -static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { - SQuery *pQuery = pRuntimeEnv->pQuery; - if (QUERY_IS_ASC_QUERY(pQuery)) { - assert(*start <= pQuery->current->lastKey); - } else { - assert(*start >= pQuery->current->lastKey); - } - - // if queried with value filter, do NOT forward query start position - if (pQuery->limit.offset <= 0 || pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->pFillInfo != NULL) { - return true; - } - - /* - * 1. for interval without interpolation query we forward pQuery->interval.interval at a time for - * pQuery->limit.offset times. Since hole exists, pQuery->interval.interval*pQuery->limit.offset value is - * not valid. otherwise, we only forward pQuery->limit.offset number of points - */ - assert(pRuntimeEnv->resultRowInfo.prevSKey == TSKEY_INITIAL_VAL); - - STimeWindow w = TSWINDOW_INITIALIZER; - bool ascQuery = QUERY_IS_ASC_QUERY(pQuery); - - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; - STableQueryInfo *pTableQueryInfo = pQuery->current; - - SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; - while (tsdbNextDataBlock(pRuntimeEnv->pQueryHandle)) { - tsdbRetrieveDataBlockInfo(pRuntimeEnv->pQueryHandle, &blockInfo); - - if (QUERY_IS_ASC_QUERY(pQuery)) { - if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { - getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &w); - pWindowResInfo->prevSKey = w.skey; - } - } else { - getAlignQueryTimeWindow(pQuery, blockInfo.window.ekey, pQuery->window.ekey, blockInfo.window.ekey, &w); - pWindowResInfo->prevSKey = w.skey; - } - - // the first time window - STimeWindow win = getActiveTimeWindow(pWindowResInfo, pWindowResInfo->prevSKey, pQuery); - - while (pQuery->limit.offset > 0) { - STimeWindow tw = win; - - if ((win.ekey <= blockInfo.window.ekey && ascQuery) || (win.ekey >= blockInfo.window.skey && !ascQuery)) { - pQuery->limit.offset -= 1; - pWindowResInfo->prevSKey = win.skey; - - // current time window is aligned with blockInfo.window.ekey - // restart it from next data block by set prevSKey to be TSKEY_INITIAL_VAL; - if ((win.ekey == blockInfo.window.ekey && ascQuery) || (win.ekey == blockInfo.window.skey && !ascQuery)) { - pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; - } - } - - if (pQuery->limit.offset == 0) { - *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo); - return true; - } - - // current window does not ended in current data block, try next data block - getNextTimeWindow(pQuery, &tw); - - /* - * If the next time window still starts from current data block, - * load the primary timestamp column first, and then find the start position for the next queried time window. - * Note that only the primary timestamp column is required. - * TODO: Optimize for this cases. All data blocks are not needed to be loaded, only if the first actually required - * time window resides in current data block. - */ - if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) { - - SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); - SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); - - if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) { - pQuery->limit.offset -= 1; - } - - if (pQuery->limit.offset == 0) { - *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo); - return true; - } else { - tw = win; - int32_t startPos = - getNextQualifiedWindow(pRuntimeEnv, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey, -1); - assert(startPos >= 0); - - // set the abort info - pQuery->pos = startPos; - pTableQueryInfo->lastKey = ((TSKEY *)pColInfoData->pData)[startPos]; - pWindowResInfo->prevSKey = tw.skey; - win = tw; - } - } else { - break; // offset is not 0, and next time window begins or ends in the next block. - } - } - } - - // check for error - if (terrno != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, terrno); - } - - return true; -} +//static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { +// SQuery *pQuery = pRuntimeEnv->pQuery; +// STableQueryInfo* pTableQueryInfo = pQuery->current; +// +// int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); +// +// if (pQuery->limit.offset == pBlockInfo->rows) { // current block will ignore completed +// pTableQueryInfo->lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pBlockInfo->window.ekey + step : pBlockInfo->window.skey + step; +// pQuery->limit.offset = 0; +// return; +// } +// +// if (QUERY_IS_ASC_QUERY(pQuery)) { +// pQuery->pos = (int32_t)pQuery->limit.offset; +// } else { +// pQuery->pos = pBlockInfo->rows - (int32_t)pQuery->limit.offset - 1; +// } +// +// assert(pQuery->pos >= 0 && pQuery->pos <= pBlockInfo->rows - 1); +// +// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); +// +// // update the pQuery->limit.offset value, and pQuery->pos value +// TSKEY *keys = (TSKEY *) pColInfoData->pData; +// +// // update the offset value +// pTableQueryInfo->lastKey = keys[pQuery->pos]; +// pQuery->limit.offset = 0; +// +// int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); +// +// qDebug("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%"PRId64, pRuntimeEnv->qinfo, +// pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, pQuery->current->lastKey); +//} + +//void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) { +// SQuery *pQuery = pRuntimeEnv->pQuery; +// +// if (pQuery->limit.offset <= 0 || pQuery->numOfFilterCols > 0) { +// return; +// } +// +// pQuery->pos = 0; +// int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); +// +// STableQueryInfo* pTableQueryInfo = pQuery->current; +// TsdbQueryHandleT pQueryHandle = pRuntimeEnv->pQueryHandle; +// +// SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; +// while (tsdbNextDataBlock(pQueryHandle)) { +// if (isQueryKilled(pRuntimeEnv->qinfo)) { +// longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); +// } +// +// tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); +// +// if (pQuery->limit.offset > blockInfo.rows) { +// pQuery->limit.offset -= blockInfo.rows; +// pTableQueryInfo->lastKey = (QUERY_IS_ASC_QUERY(pQuery)) ? blockInfo.window.ekey : blockInfo.window.skey; +// pTableQueryInfo->lastKey += step; +// +// qDebug("QInfo:%p skip rows:%d, offset:%" PRId64, pRuntimeEnv->qinfo, blockInfo.rows, +// pQuery->limit.offset); +// } else { // find the appropriated start position in current block +// updateOffsetVal(pRuntimeEnv, &blockInfo); +// break; +// } +// } +// +// if (terrno != TSDB_CODE_SUCCESS) { +// longjmp(pRuntimeEnv->env, terrno); +// } +//} + +//static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* win, SDataBlockInfo* pBlockInfo, STableQueryInfo* pTableQueryInfo) { +// SQuery *pQuery = pRuntimeEnv->pQuery; +// SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; +// +// assert(pQuery->limit.offset == 0); +// STimeWindow tw = *win; +// getNextTimeWindow(pQuery, &tw); +// +// if ((tw.skey <= pBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || +// (tw.ekey >= pBlockInfo->window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { +// +// // load the data block and check data remaining in current data block +// // TODO optimize performance +// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); +// +// tw = *win; +// int32_t startPos = +// getNextQualifiedWindow(pQuery, &tw, pBlockInfo, pColInfoData->pData, binarySearchForKey, -1); +// assert(startPos >= 0); +// +// // set the abort info +// pQuery->pos = startPos; +// +// // reset the query start timestamp +// pTableQueryInfo->win.skey = ((TSKEY *)pColInfoData->pData)[startPos]; +// pQuery->window.skey = pTableQueryInfo->win.skey; +// TSKEY key = pTableQueryInfo->win.skey; +// +// pWindowResInfo->prevSKey = tw.skey; +// int32_t index = pRuntimeEnv->resultRowInfo.curIndex; +// +// int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); +// pRuntimeEnv->resultRowInfo.curIndex = index; // restore the window index +// +// qDebug("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%" PRId64, +// pRuntimeEnv->qinfo, pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, +// pQuery->current->lastKey); +// +// return key; +// } else { // do nothing +// pQuery->window.skey = tw.skey; +// pWindowResInfo->prevSKey = tw.skey; +// pTableQueryInfo->lastKey = tw.skey; +// +// return tw.skey; +// } +// +// return true; +//} + +//static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { +// SQuery *pQuery = pRuntimeEnv->pQuery; +// if (QUERY_IS_ASC_QUERY(pQuery)) { +// assert(*start <= pQuery->current->lastKey); +// } else { +// assert(*start >= pQuery->current->lastKey); +// } +// +// // if queried with value filter, do NOT forward query start position +// if (pQuery->limit.offset <= 0 || pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->pFillInfo != NULL) { +// return true; +// } +// +// /* +// * 1. for interval without interpolation query we forward pQuery->interval.interval at a time for +// * pQuery->limit.offset times. Since hole exists, pQuery->interval.interval*pQuery->limit.offset value is +// * not valid. otherwise, we only forward pQuery->limit.offset number of points +// */ +// assert(pRuntimeEnv->resultRowInfo.prevSKey == TSKEY_INITIAL_VAL); +// +// STimeWindow w = TSWINDOW_INITIALIZER; +// bool ascQuery = QUERY_IS_ASC_QUERY(pQuery); +// +// SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; +// STableQueryInfo *pTableQueryInfo = pQuery->current; +// +// SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; +// while (tsdbNextDataBlock(pRuntimeEnv->pQueryHandle)) { +// tsdbRetrieveDataBlockInfo(pRuntimeEnv->pQueryHandle, &blockInfo); +// +// if (QUERY_IS_ASC_QUERY(pQuery)) { +// if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { +// getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &w); +// pWindowResInfo->prevSKey = w.skey; +// } +// } else { +// getAlignQueryTimeWindow(pQuery, blockInfo.window.ekey, pQuery->window.ekey, blockInfo.window.ekey, &w); +// pWindowResInfo->prevSKey = w.skey; +// } +// +// // the first time window +// STimeWindow win = getActiveTimeWindow(pWindowResInfo, pWindowResInfo->prevSKey, pQuery); +// +// while (pQuery->limit.offset > 0) { +// STimeWindow tw = win; +// +// if ((win.ekey <= blockInfo.window.ekey && ascQuery) || (win.ekey >= blockInfo.window.skey && !ascQuery)) { +// pQuery->limit.offset -= 1; +// pWindowResInfo->prevSKey = win.skey; +// +// // current time window is aligned with blockInfo.window.ekey +// // restart it from next data block by set prevSKey to be TSKEY_INITIAL_VAL; +// if ((win.ekey == blockInfo.window.ekey && ascQuery) || (win.ekey == blockInfo.window.skey && !ascQuery)) { +// pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; +// } +// } +// +// if (pQuery->limit.offset == 0) { +// *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo); +// return true; +// } +// +// // current window does not ended in current data block, try next data block +// getNextTimeWindow(pQuery, &tw); +// +// /* +// * If the next time window still starts from current data block, +// * load the primary timestamp column first, and then find the start position for the next queried time window. +// * Note that only the primary timestamp column is required. +// * TODO: Optimize for this cases. All data blocks are not needed to be loaded, only if the first actually required +// * time window resides in current data block. +// */ +// if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) { +// +// SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); +// +// if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) { +// pQuery->limit.offset -= 1; +// } +// +// if (pQuery->limit.offset == 0) { +// *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo); +// return true; +// } else { +// tw = win; +// int32_t startPos = +// getNextQualifiedWindow(pQuery, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey, -1); +// assert(startPos >= 0); +// +// // set the abort info +// pQuery->pos = startPos; +// pTableQueryInfo->lastKey = ((TSKEY *)pColInfoData->pData)[startPos]; +// pWindowResInfo->prevSKey = tw.skey; +// win = tw; +// } +// } else { +// break; // offset is not 0, and next time window begins or ends in the next block. +// } +// } +// } +// +// // check for error +// if (terrno != TSDB_CODE_SUCCESS) { +// longjmp(pRuntimeEnv->env, terrno); +// } +// +// return true; +//} static void doDestroyTableQueryInfo(STableGroupInfo* pTableqinfoGroupInfo); @@ -4538,40 +3854,40 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + // TODO set the tags scan handle if (onlyQueryTags(pQuery)) { return TSDB_CODE_SUCCESS; } - if (isSTableQuery && (!QUERY_IS_INTERVAL_QUERY(pQuery)) && (!isFixedOutputQuery(pRuntimeEnv))) { - return TSDB_CODE_SUCCESS; - } - STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); + if (isTsCompQuery(pQuery) || isPointInterpoQuery(pQuery)) { + cond.type = BLOCK_LOAD_TABLE_SEQ_ORDER; + } if (!isSTableQuery - && (pQInfo->tableqinfoGroupInfo.numOfTables == 1) + && (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 1) && (cond.order == TSDB_ORDER_ASC) && (!QUERY_IS_INTERVAL_QUERY(pQuery)) && (!isGroupbyColumn(pQuery->pGroupbyExpr)) - && (!isFixedOutputQuery(pRuntimeEnv)) + && (!isFixedOutputQuery(pQuery)) ) { - SArray* pa = GET_TABLEGROUP(pQInfo, 0); + SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0); STableQueryInfo* pCheckInfo = taosArrayGetP(pa, 0); cond.twindow = pCheckInfo->win; } terrno = TSDB_CODE_SUCCESS; if (isFirstLastRowQuery(pQuery)) { - pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); + pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQuery->tableGroupInfo, pQInfo, &pQuery->memRef); // update the query time window pQuery->window = cond.twindow; - if (pQInfo->tableGroupInfo.numOfTables == 0) { - pQInfo->tableqinfoGroupInfo.numOfTables = 0; + if (pQuery->tableGroupInfo.numOfTables == 0) { + pRuntimeEnv->tableqinfoGroupInfo.numOfTables = 0; } else { - size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo); + size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); for(int32_t i = 0; i < numOfGroups; ++i) { - SArray *group = GET_TABLEGROUP(pQInfo, i); + SArray *group = GET_TABLEGROUP(pRuntimeEnv, i); size_t t = taosArrayGetSize(group); for (int32_t j = 0; j < t; ++j) { @@ -4583,26 +3899,24 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) } } } else if (isPointInterpoQuery(pQuery)) { - pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); + pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQuery->tableGroupInfo, pQInfo, &pQuery->memRef); } else { - pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); + pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQuery->tableGroupInfo, pQInfo, &pQuery->memRef); } return terrno; } -static SFillColInfo* createFillColInfo(SQuery* pQuery) { - int32_t numOfCols = getNumOfFinalResCol(pQuery); +static SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, int64_t* fillVal) { int32_t offset = 0; - SFillColInfo* pFillCol = calloc(numOfCols, sizeof(SFillColInfo)); + SFillColInfo* pFillCol = calloc(numOfOutput, sizeof(SFillColInfo)); if (pFillCol == NULL) { return NULL; } - // TODO refactor - for(int32_t i = 0; i < numOfCols; ++i) { - SExprInfo* pExprInfo = (pQuery->pExpr2 == NULL)? &pQuery->pExpr1[i]:&pQuery->pExpr2[i]; + for(int32_t i = 0; i < numOfOutput; ++i) { + SExprInfo* pExprInfo = &pExpr[i]; pFillCol[i].col.bytes = pExprInfo->bytes; pFillCol[i].col.type = (int8_t)pExprInfo->type; @@ -4610,7 +3924,7 @@ static SFillColInfo* createFillColInfo(SQuery* pQuery) { pFillCol[i].tagIndex = -2; pFillCol[i].flag = TSDB_COL_NORMAL; // always be ta normal column for table query pFillCol[i].functionId = pExprInfo->base.functionId; - pFillCol[i].fillVal.i = pQuery->fillVal[i]; + pFillCol[i].fillVal.i = fillVal[i]; offset += pExprInfo->bytes; } @@ -4622,11 +3936,15 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, SArray* prevResult, void *ts SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + pQuery->tsdb = tsdb; + + pQuery->topBotQuery = isTopBottomQuery(pQuery); + pQuery->hasTagResults = hasTagValOutput(pQuery); + pQuery->timeWindowInterpo = timeWindowInterpoRequired(pQuery); + pQuery->stabledev = isStabledev(pQuery); - pRuntimeEnv->topBotQuery = isTopBottomQuery(pQuery); - pRuntimeEnv->hasTagResults = hasTagValOutput(pQuery); - pRuntimeEnv->timeWindowInterpo = timeWindowInterpoRequired(pQuery); pRuntimeEnv->prevResult = prevResult; + pRuntimeEnv->qinfo = pQInfo; setScanLimitationByResultBuffer(pQuery); @@ -4635,17 +3953,31 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, SArray* prevResult, void *ts return code; } - pQInfo->tsdb = tsdb; - pQInfo->vgId = vgId; - pQInfo->groupResInfo.totalGroup = (int32_t) (isSTableQuery? GET_NUM_OF_TABLEGROUP(pQInfo):0); + pQuery->tsdb = tsdb; + pQuery->vgId = vgId; + pQuery->stableQuery = isSTableQuery; + pQuery->groupbyColumn = isGroupbyColumn(pQuery->pGroupbyExpr); + pQuery->interBufSize = getOutputInterResultBufSize(pQuery); + + pRuntimeEnv->groupResInfo.totalGroup = (int32_t) (isSTableQuery? GET_NUM_OF_TABLEGROUP(pRuntimeEnv):0); pRuntimeEnv->pQuery = pQuery; pRuntimeEnv->pTsBuf = pTsBuf; pRuntimeEnv->cur.vgroupIndex = -1; - pRuntimeEnv->stableQuery = isSTableQuery; - pRuntimeEnv->prevGroupId = INT32_MIN; - pRuntimeEnv->groupbyColumn = isGroupbyColumn(pQuery->pGroupbyExpr); - pRuntimeEnv->stabledev = isStabledev(pQuery); + setResultBufSize(pQuery, &pRuntimeEnv->resultInfo); + + if (onlyQueryTags(pQuery)) { + pRuntimeEnv->resultInfo.capacity = 4096; + pRuntimeEnv->proot = createTagScanOperatorInfo(pRuntimeEnv, pQuery->pExpr1, pQuery->numOfOutput); + } else if (pQuery->queryBlockDist) { + pRuntimeEnv->pTableScanner = createTableBlockInfoScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); + } else if (isTsCompQuery(pQuery) || isPointInterpoQuery(pQuery)) { + pRuntimeEnv->pTableScanner = createTableSeqScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); + } else if (needReverseScan(pQuery)) { + pRuntimeEnv->pTableScanner = createDataBlocksOptScanInfo(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQuery), 1); + } else { + pRuntimeEnv->pTableScanner = createTableScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQuery)); + } if (pTsBuf != NULL) { int16_t order = (pQuery->order.order == pRuntimeEnv->pTsBuf->tsOrder) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; @@ -4653,109 +3985,26 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, SArray* prevResult, void *ts } int32_t ps = DEFAULT_PAGE_SIZE; - int32_t rowsize = 0; - getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize); - int32_t TENMB = 1024*1024*10; - - if (isSTableQuery && !onlyQueryTags(pRuntimeEnv->pQuery)) { - code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TENMB, pQInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - if (!QUERY_IS_INTERVAL_QUERY(pQuery)) { - int16_t type = TSDB_DATA_TYPE_NULL; - if (pRuntimeEnv->groupbyColumn) { // group by columns not tags; - type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); - } else { - type = TSDB_DATA_TYPE_INT; // group id - } - - code = initResultRowInfo(&pRuntimeEnv->resultRowInfo, 8, type); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - } else if (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery) || (!isSTableQuery)) { - int32_t numOfResultRows = getInitialPageNum(pQInfo); - getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize); - code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TENMB, pQInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - int16_t type = TSDB_DATA_TYPE_NULL; - if (pRuntimeEnv->groupbyColumn) { - type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); - } else { - type = TSDB_DATA_TYPE_TIMESTAMP; - } + getIntermediateBufInfo(pRuntimeEnv, &ps, &pQuery->intermediateResultRowSize); - code = initResultRowInfo(&pRuntimeEnv->resultRowInfo, numOfResultRows, type); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + int32_t TENMB = 1024*1024*10; + code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, ps, TENMB, pQInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; } // create runtime environment - code = setupQueryRuntimeEnv(pRuntimeEnv, (int32_t) pQInfo->tableGroupInfo.numOfTables, pQuery->order.order, pQInfo->vgId); + int32_t numOfTables = (int32_t)pQuery->tableGroupInfo.numOfTables; + pQInfo->summary.tableInfoSize += (numOfTables * sizeof(STableQueryInfo)); + code = setupQueryRuntimeEnv(pRuntimeEnv, (int32_t) pQuery->tableGroupInfo.numOfTables); if (code != TSDB_CODE_SUCCESS) { return code; } - if (pQuery->fillType != TSDB_FILL_NONE && !isPointInterpoQuery(pQuery)) { - SFillColInfo* pColInfo = createFillColInfo(pQuery); - STimeWindow w = TSWINDOW_INITIALIZER; - - TSKEY sk = MIN(pQuery->window.skey, pQuery->window.ekey); - TSKEY ek = MAX(pQuery->window.skey, pQuery->window.ekey); - getAlignQueryTimeWindow(pQuery, pQuery->window.skey, sk, ek, &w); - - int32_t numOfCols = getNumOfFinalResCol(pQuery); - pRuntimeEnv->pFillInfo = taosCreateFillInfo(pQuery->order.order, w.skey, 0, (int32_t)pQuery->rec.capacity, numOfCols, - pQuery->interval.sliding, pQuery->interval.slidingUnit, (int8_t)pQuery->precision, - pQuery->fillType, pColInfo, pQInfo); - } - - setQueryStatus(pQuery, QUERY_NOT_COMPLETED); + setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); return TSDB_CODE_SUCCESS; } -static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[i]); - if (pResInfo != NULL) { - pResInfo->complete = false; - } - } -} - -static FORCE_INLINE void setEnvForEachBlock(SQInfo* pQInfo, STableQueryInfo* pTableQueryInfo, SDataBlockInfo* pBlockInfo) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - - if (pRuntimeEnv->hasTagResults || pRuntimeEnv->pTsBuf != NULL) { - setTagVal(pRuntimeEnv, pTableQueryInfo->pTable); - } - - if (pRuntimeEnv->pTsBuf != NULL) { - setTimestampListJoinInfo(pQInfo, pTableQueryInfo); - } - - if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - if (!QUERY_IS_ASC_QUERY(pRuntimeEnv->pQuery)) { - setIntervalQueryRange(pQInfo, pBlockInfo->window.ekey); - } else { - setIntervalQueryRange(pQInfo, pBlockInfo->window.skey); - } - } else { // non-interval query - setExecutionContext(pQInfo, pTableQueryInfo->groupIndex, pBlockInfo->window.ekey + step); - } -} - static void doTableQueryInfoTimeWindowCheck(SQuery* pQuery, STableQueryInfo* pTableQueryInfo) { if (QUERY_IS_ASC_QUERY(pQuery)) { assert( @@ -4770,1123 +4019,1297 @@ static void doTableQueryInfoTimeWindowCheck(SQuery* pQuery, STableQueryInfo* pTa } } -static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery* pQuery = pRuntimeEnv->pQuery; - SQueryCostInfo* summary = &pRuntimeEnv->summary; - - int64_t st = taosGetTimestampMs(); - - TsdbQueryHandleT pQueryHandle = IS_MASTER_SCAN(pRuntimeEnv)? pRuntimeEnv->pQueryHandle : pRuntimeEnv->pSecQueryHandle; - SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; - - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); +STsdbQueryCond createTsdbQueryCond(SQuery* pQuery, STimeWindow* win) { + STsdbQueryCond cond = { + .colList = pQuery->colList, + .order = pQuery->order.order, + .numOfCols = pQuery->numOfCols, + .type = BLOCK_LOAD_OFFSET_SEQ_ORDER, + .loadExternalRows = false, + }; - while (tsdbNextDataBlock(pQueryHandle)) { - summary->totalBlocks += 1; + TIME_WINDOW_COPY(cond.twindow, *win); + return cond; +} - if (isQueryKilled(pQInfo)) { - longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); - } +static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo) { + STableIdInfo tidInfo; + STableId* id = TSDB_TABLEID(pTableQueryInfo->pTable); - tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); - STableQueryInfo **pTableQueryInfo = (STableQueryInfo**) taosHashGet(pQInfo->tableqinfoGroupInfo.map, &blockInfo.tid, sizeof(blockInfo.tid)); - if(pTableQueryInfo == NULL) { - break; - } + tidInfo.uid = id->uid; + tidInfo.tid = id->tid; + tidInfo.key = pTableQueryInfo->lastKey; - pQuery->current = *pTableQueryInfo; - doTableQueryInfoTimeWindowCheck(pQuery, *pTableQueryInfo); + return tidInfo; +} - if (!pRuntimeEnv->groupbyColumn) { - setEnvForEachBlock(pQInfo, *pTableQueryInfo, &blockInfo); - } +static void updateTableIdInfo(STableQueryInfo* pTableQueryInfo, SSDataBlock* pBlock, SHashObj* pTableIdInfo, int32_t order) { + int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); + pTableQueryInfo->lastKey = ((order == TSDB_ORDER_ASC)? pBlock->info.window.ekey:pBlock->info.window.skey) + step; - if (pRuntimeEnv->stabledev) { - for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { - if (pQuery->pExpr1[i].base.functionId == TSDB_FUNC_STDDEV_DST) { - setParamValue(pRuntimeEnv); - break; - } - } + STableIdInfo tidInfo = createTableIdInfo(pTableQueryInfo); + STableIdInfo *idinfo = taosHashGet(pTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid)); + if (idinfo != NULL) { + assert(idinfo->tid == tidInfo.tid && idinfo->uid == tidInfo.uid); + idinfo->key = tidInfo.key; + } else { + taosHashPut(pTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid), &tidInfo, sizeof(STableIdInfo)); + } +} + +static void doCloseAllTimeWindow(SQueryRuntimeEnv* pRuntimeEnv) { + size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); + for (int32_t i = 0; i < numOfGroup; ++i) { + SArray* group = GET_TABLEGROUP(pRuntimeEnv, i); + + size_t num = taosArrayGetSize(group); + for (int32_t j = 0; j < num; ++j) { + STableQueryInfo* item = taosArrayGetP(group, j); + closeAllResultRows(&item->resInfo); } + } +} - uint32_t status = 0; - SDataStatis *pStatis = NULL; - SArray *pDataBlock = NULL; +static SSDataBlock* doTableScanImpl(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; - int32_t ret = loadDataBlockOnDemand(pRuntimeEnv, &pQuery->current->resInfo, pQueryHandle, &blockInfo, &pStatis, &pDataBlock, &status); - if (ret != TSDB_CODE_SUCCESS) { - break; + STableScanInfo* pTableScanInfo = pOperator->info; + SSDataBlock* pBlock = &pTableScanInfo->block; + SQuery* pQuery = pOperator->pRuntimeEnv->pQuery; + STableGroupInfo* pTableGroupInfo = &pOperator->pRuntimeEnv->tableqinfoGroupInfo; + + while (tsdbNextDataBlock(pTableScanInfo->pQueryHandle)) { + if (isQueryKilled(pOperator->pRuntimeEnv->qinfo)) { + longjmp(pOperator->pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } - if (status == BLK_DATA_DISCARD) { - pQuery->current->lastKey = QUERY_IS_ASC_QUERY(pQuery)? blockInfo.window.ekey + step : blockInfo.window.skey + step; - continue; + pTableScanInfo->numOfBlocks += 1; + tsdbRetrieveDataBlockInfo(pTableScanInfo->pQueryHandle, &pBlock->info); + + // todo opt + if (pTableGroupInfo->numOfTables > 1 || (pQuery->current == NULL && pTableGroupInfo->numOfTables == 1)) { + STableQueryInfo** pTableQueryInfo = + (STableQueryInfo**)taosHashGet(pTableGroupInfo->map, &pBlock->info.tid, sizeof(pBlock->info.tid)); + if (pTableQueryInfo == NULL) { + break; + } + + pQuery->current = *pTableQueryInfo; + doTableQueryInfoTimeWindowCheck(pQuery, *pTableQueryInfo); } - summary->totalRows += blockInfo.rows; - stableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, pDataBlock, binarySearchForKey); + // this function never returns error? + uint32_t status; + int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pOperator->pRuntimeEnv->env, code); + } - qDebug("QInfo:%p check data block completed, uid:%"PRId64", tid:%d, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, " - "lastKey:%" PRId64, - pQInfo, blockInfo.uid, blockInfo.tid, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, - pQuery->current->lastKey); - } + // current block is ignored according to filter result by block statistics data, continue load the next block + if (status == BLK_DATA_DISCARD || pBlock->info.rows == 0) { + continue; + } - if (terrno != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, terrno); + return pBlock; } - updateWindowResNumOfRes(pRuntimeEnv); - - int64_t et = taosGetTimestampMs(); - return et - st; + return NULL; } -static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; +static SSDataBlock* doTableScan(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; - setQueryStatus(pQuery, QUERY_NOT_COMPLETED); - SArray *group = GET_TABLEGROUP(pQInfo, 0); - STableQueryInfo* pCheckInfo = taosArrayGetP(group, index); + STableScanInfo *pTableScanInfo = pOperator->info; + SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; + SQuery* pQuery = pRuntimeEnv->pQuery; - if (pRuntimeEnv->hasTagResults || pRuntimeEnv->pTsBuf != NULL) { - setTagVal(pRuntimeEnv, pCheckInfo->pTable); - } + SResultRowInfo* pResultRowInfo = pTableScanInfo->pResultRowInfo; - STableId* id = TSDB_TABLEID(pCheckInfo->pTable); - qDebug("QInfo:%p query on (%d): uid:%" PRIu64 ", tid:%d, qrange:%" PRId64 "-%" PRId64, pQInfo, index, - id->uid, id->tid, pCheckInfo->lastKey, pCheckInfo->win.ekey); + while (pTableScanInfo->current < pTableScanInfo->times) { + SSDataBlock* p = doTableScanImpl(pOperator); + if (p != NULL) { + return p; + } - STsdbQueryCond cond = { - .twindow = {pCheckInfo->lastKey, pCheckInfo->win.ekey}, - .order = pQuery->order.order, - .colList = pQuery->colList, - .numOfCols = pQuery->numOfCols, - .loadExternalRows = false, - }; + if (++pTableScanInfo->current >= pTableScanInfo->times) { + if (pTableScanInfo->reverseTimes <= 0) { + return NULL; + } else { + break; + } + } - // todo refactor - SArray *g1 = taosArrayInit(1, POINTER_BYTES); - SArray *tx = taosArrayInit(1, sizeof(STableKeyInfo)); + // do prepare for the next round table scan operation + STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); + tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); - STableKeyInfo info = {.pTable = pCheckInfo->pTable, .lastKey = pCheckInfo->lastKey}; - taosArrayPush(tx, &info); + setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); + pRuntimeEnv->scanFlag = REPEAT_SCAN; - taosArrayPush(g1, &tx); - STableGroupInfo gp = {.numOfTables = 1, .pGroupList = g1}; + if (pRuntimeEnv->pTsBuf) { + bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); + assert(ret); + } - // include only current table - if (pRuntimeEnv->pQueryHandle != NULL) { - tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); - pRuntimeEnv->pQueryHandle = NULL; - } + pResultRowInfo->curIndex = 0; + pResultRowInfo->prevSKey = pResultRowInfo->pResult[0]->win.skey; - pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo, &pQInfo->memRef); - taosArrayDestroy(tx); - taosArrayDestroy(g1); - if (pRuntimeEnv->pQueryHandle == NULL) { - longjmp(pRuntimeEnv->env, terrno); + qDebug("QInfo:%p start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, + pRuntimeEnv->qinfo, cond.twindow.skey, cond.twindow.ekey); } - if (pRuntimeEnv->pTsBuf != NULL) { - tVariant* pTag = &pRuntimeEnv->pCtx[0].tag; - - if (pRuntimeEnv->cur.vgroupIndex == -1) { - STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTsBuf, pQInfo->vgId, pTag); - // failed to find data with the specified tag value and vnodeId - if (!tsBufIsValidElem(&elem)) { - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz); - } else { - qError("QInfo:%p failed to find tag:%"PRId64" in ts_comp", pQInfo, pTag->i64); - } + if (pTableScanInfo->reverseTimes > 0) { + setupEnvForReverseScan(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); - return false; - } else { - STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); + STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); + tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, - cur.blockIndex, cur.tsIndex); - } else { - qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64, - cur.blockIndex, cur.tsIndex); - } - } - } else { - STSElem elem = tsBufGetElem(pRuntimeEnv->pTsBuf); - if (tVariantCompare(elem.tag, &pRuntimeEnv->pCtx[0].tag) != 0) { + qDebug("QInfo:%p start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, + pRuntimeEnv->qinfo, cond.twindow.skey, cond.twindow.ekey); - STSElem elem1 = tsBufGetElemStartPos(pRuntimeEnv->pTsBuf, pQInfo->vgId, pTag); - // failed to find data with the specified tag value and vnodeId - if (!tsBufIsValidElem(&elem1)) { - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz); - } else { - qError("QInfo:%p failed to find tag:%"PRId64" in ts_comp", pQInfo, pTag->i64); - } + pRuntimeEnv->scanFlag = REVERSE_SCAN; - return false; - } else { - STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, cur.blockIndex, cur.tsIndex); - } else { - qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64, cur.blockIndex, cur.tsIndex); - } - } + pTableScanInfo->times = 1; + pTableScanInfo->current = 0; + pTableScanInfo->reverseTimes = 0; + pTableScanInfo->order = cond.order; - } else { - tsBufSetCursor(pRuntimeEnv->pTsBuf, &pRuntimeEnv->cur); - STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qDebug("QInfo:%p continue scan ts_comp file, tag:%s blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, cur.blockIndex, cur.tsIndex); - } else { - qDebug("QInfo:%p continue scan ts_comp file, tag:%"PRId64" blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64, cur.blockIndex, cur.tsIndex); - } - } + if (pResultRowInfo->size > 0) { + pResultRowInfo->curIndex = pResultRowInfo->size-1; + pResultRowInfo->prevSKey = pResultRowInfo->pResult[pResultRowInfo->size-1]->win.skey; + } + + SSDataBlock* p = doTableScanImpl(pOperator); + if (p != NULL) { + return p; } } - initCtxOutputBuf(pRuntimeEnv); - return true; + return NULL; } -STsdbQueryCond createTsdbQueryCond(SQuery* pQuery, STimeWindow* win) { - STsdbQueryCond cond = { - .colList = pQuery->colList, - .order = pQuery->order.order, - .numOfCols = pQuery->numOfCols, - .loadExternalRows = false, - }; +static SSDataBlock* doBlockInfoScan(void* param) { + SOperatorInfo *pOperator = (SOperatorInfo*)param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } - TIME_WINDOW_COPY(cond.twindow, *win); - return cond; -} + STableScanInfo *pTableScanInfo = pOperator->info; -static STableIdInfo createTableIdInfo(SQuery* pQuery) { - assert(pQuery != NULL && pQuery->current != NULL); + STableBlockDist tableBlockDist = {0}; + tableBlockDist.numOfTables = (int32_t)pOperator->pRuntimeEnv->tableqinfoGroupInfo.numOfTables; + tableBlockDist.dataBlockInfos = taosArrayInit(512, sizeof(SFileBlockInfo)); - STableIdInfo tidInfo; - STableId* id = TSDB_TABLEID(pQuery->current->pTable); + tsdbGetFileBlocksDistInfo(pTableScanInfo->pQueryHandle, &tableBlockDist); + tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pQueryHandle); - tidInfo.uid = id->uid; - tidInfo.tid = id->tid; - tidInfo.key = pQuery->current->lastKey; + SSDataBlock* pBlock = &pTableScanInfo->block; + pBlock->info.rows = 1; + pBlock->info.numOfCols = 1; - return tidInfo; -} + SBufferWriter bw = tbufInitWriter(NULL, false); + blockDistInfoToBinary(&tableBlockDist, &bw); + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0); -static void updateTableIdInfo(SQuery* pQuery, SHashObj* pTableIdInfo) { - STableIdInfo tidInfo = createTableIdInfo(pQuery); - STableIdInfo* idinfo = taosHashGet(pTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid)); - if (idinfo != NULL) { - assert(idinfo->tid == tidInfo.tid && idinfo->uid == tidInfo.uid); - idinfo->key = tidInfo.key; - } else { - taosHashPut(pTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid), &tidInfo, sizeof(STableIdInfo)); - } -} + int32_t len = (int32_t) tbufTell(&bw); + pColInfo->pData = malloc(len + sizeof(int32_t)); -/** - * super table query handler - * 1. super table projection query, group-by on normal columns query, ts-comp query - * 2. point interpolation query, last row query - * - * @param pQInfo - */ -static void sequentialTableProcess(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; - setQueryStatus(pQuery, QUERY_COMPLETED); + *(int32_t*) pColInfo->pData = len; + memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len); - size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo); + tbufCloseWriter(&bw); - if (isPointInterpoQuery(pQuery)) { - resetDefaultResInfoOutputBuf(pRuntimeEnv); - assert(pQuery->limit.offset == 0 && pQuery->limit.limit != 0); + SArray* g = GET_TABLEGROUP(pOperator->pRuntimeEnv, 0); + pOperator->pRuntimeEnv->pQuery->current = taosArrayGetP(g, 0); - while (pQInfo->groupIndex < numOfGroups) { - SArray *group = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, pQInfo->groupIndex); + pOperator->status = OP_EXEC_DONE; + return pBlock; +} - qDebug("QInfo:%p point interpolation query on group:%d, total group:%" PRIzu ", current group:%p", pQInfo, - pQInfo->groupIndex, numOfGroups, group); - STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); +SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime) { + assert(repeatTime > 0); - SArray *g1 = taosArrayInit(1, POINTER_BYTES); - SArray *tx = taosArrayDup(group); - taosArrayPush(g1, &tx); + STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); + pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->times = repeatTime; + pInfo->reverseTimes = 0; + pInfo->order = pRuntimeEnv->pQuery->order.order; + pInfo->current = 0; - STableGroupInfo gp = {.numOfTables = taosArrayGetSize(tx), .pGroupList = g1}; + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "TableScanOperator"; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->numOfOutput = pRuntimeEnv->pQuery->numOfCols; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = doTableScan; - // include only current table - if (pRuntimeEnv->pQueryHandle != NULL) { - tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); - pRuntimeEnv->pQueryHandle = NULL; - } + return pOperator; +} - pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(pQInfo->tsdb, &cond, &gp, pQInfo, &pQInfo->memRef); +SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) { + STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - taosArrayDestroy(tx); - taosArrayDestroy(g1); - if (pRuntimeEnv->pQueryHandle == NULL) { - longjmp(pRuntimeEnv->env, terrno); - } + pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->times = 1; + pInfo->reverseTimes = 0; + pInfo->order = pRuntimeEnv->pQuery->order.order; + pInfo->current = 0; - initCtxOutputBuf(pRuntimeEnv); + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "TableSeqScanOperator"; + pOperator->operatorType = OP_TableSeqScan; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->numOfOutput = pRuntimeEnv->pQuery->numOfCols; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = doTableScanImpl; - SArray *s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle); - assert(taosArrayGetSize(s) >= 1); + return pOperator; +} - setTagVal(pRuntimeEnv, taosArrayGetP(s, 0)); - taosArrayDestroy(s); +SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) { + STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - // here we simply set the first table as current table - SArray *first = GET_TABLEGROUP(pQInfo, pQInfo->groupIndex); - pQuery->current = taosArrayGetP(first, 0); + pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey); + SColumnInfoData infoData = {{0}}; + infoData.info.type = TSDB_DATA_TYPE_BINARY; + infoData.info.bytes = 1024; + infoData.info.colId = TSDB_BLOCK_DIST_COLUMN_INDEX; + taosArrayPush(pInfo->block.pDataBlock, &infoData); - int64_t numOfRes = getNumOfResult(pRuntimeEnv); - if (numOfRes > 0) { - pQuery->rec.rows += numOfRes; - forwardCtxOutputBuf(pRuntimeEnv, numOfRes); - } + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "TableBlockInfoScanOperator"; + pOperator->operatorType = OP_TableBlockInfoScan; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->numOfOutput = pRuntimeEnv->pQuery->numOfCols; + pOperator->exec = doBlockInfoScan; - skipResults(pRuntimeEnv); - pQInfo->groupIndex += 1; + return pOperator; +} - // enable execution for next table, when handling the projection query - enableExecutionForNextTable(pRuntimeEnv); +void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInfo* pDownstream) { + assert(pTableScanInfo != NULL && pDownstream != NULL); - if (pQuery->rec.rows >= pQuery->rec.capacity) { - setQueryStatus(pQuery, QUERY_RESBUF_FULL); - break; - } - } - } else if (pRuntimeEnv->groupbyColumn) { // group-by on normal columns query - while (pQInfo->groupIndex < numOfGroups) { - SArray *group = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, pQInfo->groupIndex); + pTableScanInfo->pExpr = pDownstream->pExpr; // TODO refactor to use colId instead of pExpr + pTableScanInfo->numOfOutput = pDownstream->numOfOutput; - qDebug("QInfo:%p group by normal columns group:%d, total group:%" PRIzu "", pQInfo, pQInfo->groupIndex, - numOfGroups); + if (pDownstream->operatorType == OP_Aggregate || pDownstream->operatorType == OP_MultiTableAggregate) { + SAggOperatorInfo* pAggInfo = pDownstream->info; - STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); + pTableScanInfo->pCtx = pAggInfo->binfo.pCtx; + pTableScanInfo->pResultRowInfo = &pAggInfo->binfo.resultRowInfo; + pTableScanInfo->rowCellInfoOffset = pAggInfo->binfo.rowCellInfoOffset; + } else if (pDownstream->operatorType == OP_TimeWindow) { + STableIntervalOperatorInfo *pIntervalInfo = pDownstream->info; - SArray *g1 = taosArrayInit(1, POINTER_BYTES); - SArray *tx = taosArrayDup(group); - taosArrayPush(g1, &tx); + pTableScanInfo->pCtx = pIntervalInfo->pCtx; + pTableScanInfo->pResultRowInfo = &pIntervalInfo->resultRowInfo; + pTableScanInfo->rowCellInfoOffset = pIntervalInfo->rowCellInfoOffset; - STableGroupInfo gp = {.numOfTables = taosArrayGetSize(tx), .pGroupList = g1}; + } else if (pDownstream->operatorType == OP_Groupby) { + SGroupbyOperatorInfo *pGroupbyInfo = pDownstream->info; - // include only current table - if (pRuntimeEnv->pQueryHandle != NULL) { - tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); - pRuntimeEnv->pQueryHandle = NULL; - } + pTableScanInfo->pCtx = pGroupbyInfo->binfo.pCtx; + pTableScanInfo->pResultRowInfo = &pGroupbyInfo->binfo.resultRowInfo; + pTableScanInfo->rowCellInfoOffset = pGroupbyInfo->binfo.rowCellInfoOffset; - // no need to update the lastkey for each table - pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo, &pQInfo->memRef); + } else if (pDownstream->operatorType == OP_MultiTableTimeInterval) { + STableIntervalOperatorInfo *pInfo = pDownstream->info; - taosArrayDestroy(g1); - taosArrayDestroy(tx); - if (pRuntimeEnv->pQueryHandle == NULL) { - longjmp(pRuntimeEnv->env, terrno); - } + pTableScanInfo->pCtx = pInfo->pCtx; + pTableScanInfo->pResultRowInfo = &pInfo->resultRowInfo; + pTableScanInfo->rowCellInfoOffset = pInfo->rowCellInfoOffset; + + } else if (pDownstream->operatorType == OP_Arithmetic) { + SArithOperatorInfo *pInfo = pDownstream->info; + + pTableScanInfo->pCtx = pInfo->binfo.pCtx; + pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo; + pTableScanInfo->rowCellInfoOffset = pInfo->binfo.rowCellInfoOffset; + } else if (pDownstream->operatorType == OP_SessionWindow) { + SSWindowOperatorInfo* pInfo = pDownstream->info; + + pTableScanInfo->pCtx = pInfo->binfo.pCtx; + pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo; + pTableScanInfo->rowCellInfoOffset = pInfo->binfo.rowCellInfoOffset; + } else { + assert(0); + } +} - SArray *s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle); - assert(taosArrayGetSize(s) >= 1); +static SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime) { + assert(repeatTime > 0); - setTagVal(pRuntimeEnv, taosArrayGetP(s, 0)); + STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); + pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->times = repeatTime; + pInfo->reverseTimes = reverseTime; + pInfo->current = 0; + pInfo->order = pRuntimeEnv->pQuery->order.order; - // here we simply set the first table as current table - scanMultiTableDataBlocks(pQInfo); - pQInfo->groupIndex += 1; + SOperatorInfo* pOptr = calloc(1, sizeof(SOperatorInfo)); + pOptr->name = "DataBlocksOptimizedScanOperator"; + pOptr->operatorType = OP_DataBlocksOptScan; + pOptr->pRuntimeEnv = pRuntimeEnv; + pOptr->blockingOptr = false; + pOptr->info = pInfo; + pOptr->exec = doTableScan; - taosArrayDestroy(s); + return pOptr; +} - // no results generated for current group, continue to try the next group - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; - if (pWindowResInfo->size <= 0) { - continue; - } +static int32_t getTableScanOrder(STableScanInfo* pTableScanInfo) { + return pTableScanInfo->order; +} - for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - pWindowResInfo->pResult[i]->closed = true; // enable return all results for group by normal columns +// this is a blocking operator +static SSDataBlock* doAggregate(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } - SResultRow *pResult = pWindowResInfo->pResult[i]; - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - SResultRowCellInfo *pCell = getResultCell(pRuntimeEnv, pResult, j); - pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pCell->numOfRes)); - } - } + SAggOperatorInfo* pAggInfo = pOperator->info; + SOptrBasicInfo* pInfo = &pAggInfo->binfo; - qDebug("QInfo:%p generated groupby columns results %d rows for group %d completed", pQInfo, pWindowResInfo->size, - pQInfo->groupIndex); + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - pQuery->rec.rows = 0; - if (pWindowResInfo->size > pQuery->rec.capacity) { - expandBuffer(pRuntimeEnv, pWindowResInfo->size, pQInfo); - } + SQuery* pQuery = pRuntimeEnv->pQuery; + int32_t order = pQuery->order.order; - initGroupResInfo(&pQInfo->groupResInfo, &pRuntimeEnv->resultRowInfo, 0); - copyToOutputBuf(pQInfo, pWindowResInfo); - assert(pQuery->rec.rows == pWindowResInfo->size); + SOperatorInfo* upstream = pOperator->upstream; - resetResultRowInfo(pRuntimeEnv, &pRuntimeEnv->resultRowInfo); - cleanupGroupResInfo(&pQInfo->groupResInfo); + while(1) { + SSDataBlock* pBlock = upstream->exec(upstream); + if (pBlock == NULL) { break; } - } else if (pRuntimeEnv->queryWindowIdentical && pRuntimeEnv->pTsBuf == NULL && !isTsCompQuery(pQuery)) { - //super table projection query with identical query time range for all tables. - SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; - resetDefaultResInfoOutputBuf(pRuntimeEnv); - SArray *group = GET_TABLEGROUP(pQInfo, 0); - assert(taosArrayGetSize(group) == pQInfo->tableqinfoGroupInfo.numOfTables && - 1 == taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList)); + setTagValue(pOperator, pQuery->current->pTable, pInfo->pCtx, pOperator->numOfOutput); - void *pQueryHandle = pRuntimeEnv->pQueryHandle; - if (pQueryHandle == NULL) { - STsdbQueryCond con = createTsdbQueryCond(pQuery, &pQuery->window); - pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &con, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); - pQueryHandle = pRuntimeEnv->pQueryHandle; + if (upstream->operatorType == OP_DataBlocksOptScan) { + STableScanInfo* pScanInfo = upstream->info; + order = getTableScanOrder(pScanInfo); } - // skip blocks without load the actual data block from file if no filter condition present - // skipBlocks(&pQInfo->runtimeEnv); - // if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0) { - // setQueryStatus(pQuery, QUERY_COMPLETED); - // return; - // } + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); + doAggregateImpl(pOperator, pQuery->window.skey, pInfo->pCtx, pBlock); + } - if (pQuery->prjInfo.vgroupLimit != -1) { - assert(pQuery->limit.limit == -1 && pQuery->limit.offset == 0); - } else if (pQuery->limit.limit != -1) { - assert(pQuery->prjInfo.vgroupLimit == -1); - } + pOperator->status = OP_EXEC_DONE; + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); - setQueryStatus(pQuery, QUERY_NOT_COMPLETED); + finalizeQueryResult(pOperator, pInfo->pCtx, &pInfo->resultRowInfo, pInfo->rowCellInfoOffset); + pInfo->pRes->info.rows = getNumOfResult(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); - bool hasMoreBlock = true; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - SQueryCostInfo *summary = &pRuntimeEnv->summary; - while ((hasMoreBlock = tsdbNextDataBlock(pQueryHandle)) == true) { - summary->totalBlocks += 1; + return pInfo->pRes; +} - if (isQueryKilled(pQInfo)) { - longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); - } +static SSDataBlock* doSTableAggregate(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } - tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); - STableQueryInfo **pTableQueryInfo = - (STableQueryInfo **) taosHashGet(pQInfo->tableqinfoGroupInfo.map, &blockInfo.tid, sizeof(blockInfo.tid)); - if (pTableQueryInfo == NULL) { - break; - } + SAggOperatorInfo* pAggInfo = pOperator->info; + SOptrBasicInfo* pInfo = &pAggInfo->binfo; - pQuery->current = *pTableQueryInfo; - doTableQueryInfoTimeWindowCheck(pQuery, *pTableQueryInfo); + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - if (pRuntimeEnv->hasTagResults) { - setTagVal(pRuntimeEnv, pQuery->current->pTable); - } + if (pOperator->status == OP_RES_TO_RETURN) { + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->pRes); - if (pQuery->prjInfo.vgroupLimit > 0 && pQuery->current->resInfo.size > pQuery->prjInfo.vgroupLimit) { - pQuery->current->lastKey = - QUERY_IS_ASC_QUERY(pQuery) ? blockInfo.window.ekey + step : blockInfo.window.skey + step; - continue; - } + if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } - // it is a super table ordered projection query, check for the number of output for each vgroup - if (pQuery->prjInfo.vgroupLimit > 0 && pQuery->rec.rows >= pQuery->prjInfo.vgroupLimit) { - if (QUERY_IS_ASC_QUERY(pQuery) && blockInfo.window.skey >= pQuery->prjInfo.ts) { - pQuery->current->lastKey = - QUERY_IS_ASC_QUERY(pQuery) ? blockInfo.window.ekey + step : blockInfo.window.skey + step; - continue; - } else if (!QUERY_IS_ASC_QUERY(pQuery) && blockInfo.window.ekey <= pQuery->prjInfo.ts) { - pQuery->current->lastKey = - QUERY_IS_ASC_QUERY(pQuery) ? blockInfo.window.ekey + step : blockInfo.window.skey + step; - continue; - } - } + return pInfo->pRes; + } - uint32_t status = 0; - SDataStatis *pStatis = NULL; - SArray *pDataBlock = NULL; + SQuery* pQuery = pRuntimeEnv->pQuery; + int32_t order = pQuery->order.order; - int32_t ret = loadDataBlockOnDemand(pRuntimeEnv, &pQuery->current->resInfo, pQueryHandle, &blockInfo, - &pStatis, &pDataBlock, &status); - if (ret != TSDB_CODE_SUCCESS) { - break; - } + SOperatorInfo* upstream = pOperator->upstream; - if(status == BLK_DATA_DISCARD) { - pQuery->current->lastKey = - QUERY_IS_ASC_QUERY(pQuery) ? blockInfo.window.ekey + step : blockInfo.window.skey + step; - continue; - } + while(1) { + SSDataBlock* pBlock = upstream->exec(upstream); + if (pBlock == NULL) { + break; + } - ensureOutputBuffer(pRuntimeEnv, blockInfo.rows); - int64_t prev = getNumOfResult(pRuntimeEnv); + setTagValue(pOperator, pRuntimeEnv->pQuery->current->pTable, pInfo->pCtx, pOperator->numOfOutput); - pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : blockInfo.rows - 1; - int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, pDataBlock); + if (upstream->operatorType == OP_DataBlocksOptScan) { + STableScanInfo* pScanInfo = upstream->info; + order = getTableScanOrder(pScanInfo); + } - summary->totalRows += blockInfo.rows; - qDebug("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%" PRId64, - GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes, - pQuery->current->lastKey); + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); - pQuery->rec.rows = getNumOfResult(pRuntimeEnv); + TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? pBlock->info.window.ekey + 1:pBlock->info.window.skey-1; + setExecutionContext(pRuntimeEnv, pInfo, pOperator->numOfOutput, pQuery->current->groupIndex, key); + doAggregateImpl(pOperator, pQuery->window.skey, pInfo->pCtx, pBlock); + } - int64_t inc = pQuery->rec.rows - prev; - pQuery->current->resInfo.size += (int32_t) inc; + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pInfo->resultRowInfo); - // the flag may be set by tableApplyFunctionsOnBlock, clear it here - CLEAR_QUERY_STATUS(pQuery, QUERY_COMPLETED); + updateNumOfRowsInResultRows(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput, &pInfo->resultRowInfo, + pInfo->rowCellInfoOffset); - updateTableIdInfo(pQuery, pQInfo->arrTableIdInfo); + initGroupResInfo(&pRuntimeEnv->groupResInfo, &pInfo->resultRowInfo); - if (pQuery->prjInfo.vgroupLimit >= 0) { - if (((pQuery->rec.rows + pQuery->rec.total) < pQuery->prjInfo.vgroupLimit) || ((pQuery->rec.rows + pQuery->rec.total) > pQuery->prjInfo.vgroupLimit && prev < pQuery->prjInfo.vgroupLimit)) { - if (QUERY_IS_ASC_QUERY(pQuery) && pQuery->prjInfo.ts < blockInfo.window.ekey) { - pQuery->prjInfo.ts = blockInfo.window.ekey; - } else if (!QUERY_IS_ASC_QUERY(pQuery) && pQuery->prjInfo.ts > blockInfo.window.skey) { - pQuery->prjInfo.ts = blockInfo.window.skey; - } - } - } else { - // the limitation of output result is reached, set the query completed - skipResults(pRuntimeEnv); - if (limitOperator(pQuery, pQInfo)) { - SET_STABLE_QUERY_OVER(pQInfo); - break; - } - } + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->pRes); + if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } - // while the output buffer is full or limit/offset is applied, query may be paused here - if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL|QUERY_COMPLETED)) { - break; - } - } + return pInfo->pRes; +} - if (!hasMoreBlock) { - setQueryStatus(pQuery, QUERY_COMPLETED); - SET_STABLE_QUERY_OVER(pQInfo); - } - } else { - /* - * the following two cases handled here. - * 1. ts-comp query, and 2. the super table projection query with different query time range for each table. - * If the subgroup index is larger than 0, results generated by group by tbname,k is existed. - * we need to return it to client in the first place. - */ - if (hasRemainData(&pQInfo->groupResInfo)) { - copyToOutputBuf(pQInfo, &pRuntimeEnv->resultRowInfo); - pQuery->rec.total += pQuery->rec.rows; +static SSDataBlock* doArithmeticOperation(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pQuery->rec.rows > 0) { - return; - } + SArithOperatorInfo* pArithInfo = pOperator->info; + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + SOptrBasicInfo *pInfo = &pArithInfo->binfo; + + SSDataBlock* pRes = pInfo->pRes; + int32_t order = pRuntimeEnv->pQuery->order.order; + + pRes->info.rows = 0; + + while(1) { + SSDataBlock* pBlock = pOperator->upstream->exec(pOperator->upstream); + if (pBlock == NULL) { + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + break; } - // all data have returned already - if (pQInfo->tableIndex >= pQInfo->tableqinfoGroupInfo.numOfTables) { - return; + STableQueryInfo* pTableQueryInfo = pRuntimeEnv->pQuery->current; + + // todo dynamic set tags + setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfOutput); + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); + updateOutputBuf(pArithInfo, pBlock->info.rows); + + arithmeticApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); + updateTableIdInfo(pTableQueryInfo, pBlock, pRuntimeEnv->pTableRetrieveTsMap, order); + + pRes->info.rows = getNumOfResult(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); + if (pRes->info.rows >= pRuntimeEnv->resultInfo.threshold) { + break; } + } - resetDefaultResInfoOutputBuf(pRuntimeEnv); - resetResultRowInfo(pRuntimeEnv, &pRuntimeEnv->resultRowInfo); + clearNumOfRes(pInfo->pCtx, pOperator->numOfOutput); + return (pInfo->pRes->info.rows > 0)? pInfo->pRes:NULL; +} - SArray *group = GET_TABLEGROUP(pQInfo, 0); - assert(taosArrayGetSize(group) == pQInfo->tableqinfoGroupInfo.numOfTables && - 1 == taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList)); +static SSDataBlock* doLimit(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } - while (pQInfo->tableIndex < pQInfo->tableqinfoGroupInfo.numOfTables) { - if (isQueryKilled(pQInfo)) { - longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); - } + SLimitOperatorInfo* pInfo = pOperator->info; - pQuery->current = taosArrayGetP(group, pQInfo->tableIndex); - if (!multiTableMultioutputHelper(pQInfo, pQInfo->tableIndex)) { - pQInfo->tableIndex++; - continue; - } + SSDataBlock* pBlock = pOperator->upstream->exec(pOperator->upstream); + if (pBlock == NULL) { + setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); + pOperator->status = OP_EXEC_DONE; + return NULL; + } - // TODO handle the limit offset problem - if (pQuery->numOfFilterCols == 0 && pQuery->limit.offset > 0) { - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - pQInfo->tableIndex++; - continue; - } - } + if (pInfo->total + pBlock->info.rows >= pInfo->limit) { + pBlock->info.rows = (int32_t) (pInfo->limit - pInfo->total); - scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey); - skipResults(pRuntimeEnv); + pInfo->total = pInfo->limit; - // the limitation of output result is reached, set the query completed - if (limitOperator(pQuery, pQInfo)) { - SET_STABLE_QUERY_OVER(pQInfo); - break; - } + setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); + pOperator->status = OP_EXEC_DONE; + } else { + pInfo->total += pBlock->info.rows; + } - // enable execution for next table, when handling the projection query - enableExecutionForNextTable(pRuntimeEnv); - - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - /* - * query range is identical in terms of all meters involved in query, - * so we need to restore them at the *beginning* of query on each meter, - * not the consecutive query on meter on which is aborted due to buffer limitation - * to ensure that, we can reset the query range once query on a meter is completed. - */ - pQInfo->tableIndex++; - updateTableIdInfo(pQuery, pQInfo->arrTableIdInfo); - - // if the buffer is full or group by each table, we need to jump out of the loop - if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { - break; - } + return pBlock; +} - if (pRuntimeEnv->pTsBuf != NULL) { - pRuntimeEnv->cur = pRuntimeEnv->pTsBuf->cur; - } +// TODO add log +static SSDataBlock* doOffset(void* param) { + SOperatorInfo *pOperator = (SOperatorInfo *)param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } - } else { - // all data in the result buffer are skipped due to the offset, continue to retrieve data from current meter - if (pQuery->rec.rows == 0) { - assert(!Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)); - continue; - } else { - // buffer is full, wait for the next round to retrieve data from current meter - assert(Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)); - break; - } - } - } + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - if (pQInfo->tableIndex >= pQInfo->tableqinfoGroupInfo.numOfTables) { - setQueryStatus(pQuery, QUERY_COMPLETED); + while (1) { + SSDataBlock *pBlock = pOperator->upstream->exec(pOperator->upstream); + if (pBlock == NULL) { + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + pOperator->status = OP_EXEC_DONE; + return NULL; } - /* - * 1. super table projection query, group-by on normal columns query, ts-comp query - * 2. point interpolation query, last row query - * - * group-by on normal columns query and last_row query do NOT invoke the finalizer here, - * since the finalize stage will be done at the client side. - * - * projection query, point interpolation query do not need the finalizer. - * - * Only the ts-comp query requires the finalizer function to be executed here. - */ - if (isTsCompQuery(pQuery)) { - finalizeQueryResult(pRuntimeEnv); - } + if (pRuntimeEnv->currentOffset == 0) { + return pBlock; + } else if (pRuntimeEnv->currentOffset > pBlock->info.rows) { + pRuntimeEnv->currentOffset -= pBlock->info.rows; + } else { + int32_t remain = (int32_t)(pBlock->info.rows - pRuntimeEnv->currentOffset); + pBlock->info.rows = remain; - if (pRuntimeEnv->pTsBuf != NULL) { - pRuntimeEnv->cur = pRuntimeEnv->pTsBuf->cur; - } + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData *pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + + int16_t bytes = pColInfoData->info.bytes; + memmove(pColInfoData->pData, pColInfoData->pData + bytes * pRuntimeEnv->currentOffset, remain * bytes); + } - qDebug("QInfo %p numOfTables:%" PRIu64 ", index:%d, numOfGroups:%" PRIzu ", %" PRId64 - " points returned, total:%" PRId64 ", offset:%" PRId64, - pQInfo, (uint64_t)pQInfo->tableqinfoGroupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, pQuery->rec.rows, - pQuery->rec.total, pQuery->limit.offset); + pRuntimeEnv->currentOffset = 0; + return pBlock; + } } } -static int32_t doSaveContext(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; +static SSDataBlock* doIntervalAgg(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } - SET_REVERSE_SCAN_FLAG(pRuntimeEnv); - SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); - SWITCH_ORDER(pQuery->order.order); + STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; - if (pRuntimeEnv->pTsBuf != NULL) { - SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); - } + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + if (pOperator->status == OP_RES_TO_RETURN) { + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); - STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } - // clean unused handle - if (pRuntimeEnv->pSecQueryHandle != NULL) { - tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); + return pIntervalInfo->pRes; } - setQueryStatus(pQuery, QUERY_NOT_COMPLETED); - switchCtxOrder(pRuntimeEnv); - disableFuncInReverseScan(pQInfo); - setupQueryRangeForReverseScan(pQInfo); + SQuery* pQuery = pRuntimeEnv->pQuery; + int32_t order = pQuery->order.order; + STimeWindow win = pQuery->window; - pRuntimeEnv->prevGroupId = INT32_MIN; - pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); - return (pRuntimeEnv->pSecQueryHandle == NULL)? -1:0; -} + SOperatorInfo* upstream = pOperator->upstream; -static void doRestoreContext(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; + while(1) { + SSDataBlock* pBlock = upstream->exec(upstream); + if (pBlock == NULL) { + break; + } - SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); - SWITCH_ORDER(pQuery->order.order); + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQuery->order.order); + hashIntervalAgg(pOperator, &pIntervalInfo->resultRowInfo, pBlock, 0); + } - if (pRuntimeEnv->pTsBuf != NULL) { - SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); + // restore the value + pQuery->order.order = order; + pQuery->window = win; + + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pIntervalInfo->resultRowInfo); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); + + initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); + + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; } - switchCtxOrder(pRuntimeEnv); - SET_MASTER_SCAN_FLAG(pRuntimeEnv); + return pIntervalInfo->pRes->info.rows == 0? NULL:pIntervalInfo->pRes; } -static void doCloseAllTimeWindow(SQInfo *pQInfo) { - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; +static SSDataBlock* doSTableIntervalAgg(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } - if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pQInfo); - for (int32_t i = 0; i < numOfGroup; ++i) { - SArray *group = GET_TABLEGROUP(pQInfo, i); - - size_t num = taosArrayGetSize(group); - for (int32_t j = 0; j < num; ++j) { - STableQueryInfo* item = taosArrayGetP(group, j); - closeAllResultRows(&item->resInfo); - } + STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + + if (pOperator->status == OP_RES_TO_RETURN) { + copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; } - } else { // close results for group result - closeAllResultRows(&pQInfo->runtimeEnv.resultRowInfo); + + return pIntervalInfo->pRes; } -} -static void multiTableQueryProcess(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery *pQuery = pRuntimeEnv->pQuery; + SQuery* pQuery = pRuntimeEnv->pQuery; + int32_t order = pQuery->order.order; - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - copyResToQueryResultBuf(pQInfo, pQuery); - } else { - copyToOutputBuf(pQInfo, &pRuntimeEnv->resultRowInfo); + SOperatorInfo* upstream = pOperator->upstream; + + while(1) { + SSDataBlock* pBlock = upstream->exec(upstream); + if (pBlock == NULL) { + break; } - qDebug("QInfo:%p current:%"PRId64", total:%"PRId64, pQInfo, pQuery->rec.rows, pQuery->rec.total); - return; + // the pDataBlock are always the same one, no need to call this again + STableQueryInfo* pTableQueryInfo = pRuntimeEnv->pQuery->current; + + setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); + setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQuery->order.order); + setIntervalQueryRange(pRuntimeEnv, pBlock->info.window.skey); + + hashIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pTableQueryInfo->groupIndex); } - qDebug("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", order:%d, forward scan start", pQInfo, - pQuery->window.skey, pQuery->window.ekey, pQuery->order.order); + pOperator->status = OP_RES_TO_RETURN; + pQuery->order.order = order; // TODO : restore the order + doCloseAllTimeWindow(pRuntimeEnv); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + + copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } - // do check all qualified data blocks - int64_t el = scanMultiTableDataBlocks(pQInfo); - qDebug("QInfo:%p master scan completed, elapsed time: %" PRId64 "ms, reverse scan start", pQInfo, el); + return pIntervalInfo->pRes; +} - // query error occurred or query is killed, abort current execution - if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) { - qDebug("QInfo:%p query killed or error occurred, code:%s, abort", pQInfo, tstrerror(pQInfo->code)); - longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); +static SSDataBlock* doSessionWindowAgg(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; } - // close all time window results - doCloseAllTimeWindow(pQInfo); + SSWindowOperatorInfo* pWindowInfo = pOperator->info; + SOptrBasicInfo* pBInfo = &pWindowInfo->binfo; - if (needReverseScan(pQuery)) { - int32_t code = doSaveContext(pQInfo); - if (code == TSDB_CODE_SUCCESS) { - el = scanMultiTableDataBlocks(pQInfo); - qDebug("QInfo:%p reversed scan completed, elapsed time: %" PRId64 "ms", pQInfo, el); - doRestoreContext(pQInfo); - } else { - pQInfo->code = code; + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + if (pOperator->status == OP_RES_TO_RETURN) { + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); + + if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; } - } else { - qDebug("QInfo:%p no need to do reversed scan, query completed", pQInfo); + + return pBInfo->pRes; } - setQueryStatus(pQuery, QUERY_COMPLETED); + SQuery* pQuery = pRuntimeEnv->pQuery; + int32_t order = pQuery->order.order; + STimeWindow win = pQuery->window; + + SOperatorInfo* upstream = pOperator->upstream; - if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) { - qDebug("QInfo:%p query killed or error occurred, code:%s, abort", pQInfo, tstrerror(pQInfo->code)); - //TODO finalizeQueryResult may cause SEGSEV, since the memory may not allocated yet, add a cleanup function instead - longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); + while(1) { + SSDataBlock* pBlock = upstream->exec(upstream); + if (pBlock == NULL) { + break; + } + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, pQuery->order.order); + doSessionWindowAggImpl(pOperator, pWindowInfo, pBlock); } - if (QUERY_IS_INTERVAL_QUERY(pQuery) || isSumAvgRateQuery(pQuery)) { - copyResToQueryResultBuf(pQInfo, pQuery); - } else { // not a interval query - initGroupResInfo(&pQInfo->groupResInfo, &pRuntimeEnv->resultRowInfo, 0); - copyToOutputBuf(pQInfo, &pRuntimeEnv->resultRowInfo); + // restore the value + pQuery->order.order = order; + pQuery->window = win; + + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pBInfo->resultRowInfo); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + finalizeQueryResult(pOperator, pBInfo->pCtx, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); + + initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo); + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); + + if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; } - // handle the limitation of output buffer - qDebug("QInfo:%p points returned:%" PRId64 ", total:%" PRId64, pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); + return pBInfo->pRes->info.rows == 0? NULL:pBInfo->pRes; } -static char *getArithemicInputSrc(void *param, const char *name, int32_t colId) { - SArithmeticSupport *pSupport = (SArithmeticSupport *) param; - SExprInfo* pExprInfo = (SExprInfo*) pSupport->exprList; +static SSDataBlock* hashGroupbyAggregate(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SGroupbyOperatorInfo *pInfo = pOperator->info; + + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + if (pOperator->status == OP_RES_TO_RETURN) { + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); + + if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pInfo->binfo.pRes; + } + + SOperatorInfo* upstream = pOperator->upstream; - int32_t index = -1; - for (int32_t i = 0; i < pSupport->numOfCols; ++i) { - if (colId == pExprInfo[i].base.resColId) { - index = i; + while(1) { + SSDataBlock* pBlock = upstream->exec(upstream); + if (pBlock == NULL) { break; } + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pRuntimeEnv->pQuery->order.order); + setTagValue(pOperator, pRuntimeEnv->pQuery->current->pTable, pInfo->binfo.pCtx, pOperator->numOfOutput); + if (pInfo->colIndex == -1) { + pInfo->colIndex = getGroupbyColumnIndex(pRuntimeEnv->pQuery->pGroupbyExpr, pBlock); + } + + doHashGroupbyAgg(pOperator, pInfo, pBlock); } - assert(index >= 0 && index < pSupport->numOfCols); - return pSupport->data[index] + pSupport->offset * pExprInfo[index].bytes; -} + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pInfo->binfo.resultRowInfo); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); -static void doSecondaryArithmeticProcess(SQuery* pQuery) { - if (pQuery->numOfExpr2 == 0) { - return; + if (!pRuntimeEnv->pQuery->stableQuery) { // finalize include the update of result rows + finalizeQueryResult(pOperator, pInfo->binfo.pCtx, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); + } else { + updateNumOfRowsInResultRows(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); } - SArithmeticSupport arithSup = {0}; - tFilePage **data = calloc(pQuery->numOfExpr2, POINTER_BYTES); - for (int32_t i = 0; i < pQuery->numOfExpr2; ++i) { - int32_t bytes = pQuery->pExpr2[i].bytes; - data[i] = (tFilePage *)malloc((size_t)(bytes * pQuery->rec.rows) + sizeof(tFilePage)); + initGroupResInfo(&pRuntimeEnv->groupResInfo, &pInfo->binfo.resultRowInfo); + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); + + if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; } - arithSup.offset = 0; - arithSup.numOfCols = (int32_t)pQuery->numOfOutput; - arithSup.exprList = pQuery->pExpr1; - arithSup.data = calloc(arithSup.numOfCols, POINTER_BYTES); + return pInfo->binfo.pRes; +} - for (int32_t k = 0; k < arithSup.numOfCols; ++k) { - arithSup.data[k] = pQuery->sdata[k]->data; +static SSDataBlock* doFill(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; } - for (int i = 0; i < pQuery->numOfExpr2; ++i) { - SExprInfo *pExpr = &pQuery->pExpr2[i]; + SFillOperatorInfo *pInfo = pOperator->info; + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - // calculate the result from several other columns - SSqlFuncMsg* pSqlFunc = &pExpr->base; - if (pSqlFunc->functionId != TSDB_FUNC_ARITHM) { + if (taosFillHasMoreResults(pInfo->pFillInfo)) { + doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, (int32_t)pRuntimeEnv->resultInfo.capacity); + return pInfo->pRes; + } - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - if (pSqlFunc->functionId == pQuery->pExpr1[j].base.functionId && - pSqlFunc->colInfo.colId == pQuery->pExpr1[j].base.colInfo.colId) { - memcpy(data[i]->data, pQuery->sdata[j]->data, (size_t)(pQuery->pExpr1[j].bytes * pQuery->rec.rows)); - break; - } + while(1) { + SSDataBlock *pBlock = pOperator->upstream->exec(pOperator->upstream); + if (pBlock == NULL) { + if (pInfo->totalInputRows == 0) { + pOperator->status = OP_EXEC_DONE; + return NULL; } + + taosFillSetStartInfo(pInfo->pFillInfo, 0, pRuntimeEnv->pQuery->window.ekey); } else { - arithSup.pArithExpr = pExpr; - arithmeticTreeTraverse(arithSup.pArithExpr->pExpr, (int32_t)pQuery->rec.rows, data[i]->data, &arithSup, TSDB_ORDER_ASC, - getArithemicInputSrc); + pInfo->totalInputRows += pBlock->info.rows; + + int64_t ekey = Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED)?pRuntimeEnv->pQuery->window.ekey:pBlock->info.window.ekey; + + taosFillSetStartInfo(pInfo->pFillInfo, pBlock->info.rows, ekey); + taosFillSetInputDataBlock(pInfo->pFillInfo, pBlock); } - } - for (int32_t i = 0; i < pQuery->numOfExpr2; ++i) { - memcpy(pQuery->sdata[i]->data, data[i]->data, (size_t)(pQuery->pExpr2[i].bytes * pQuery->rec.rows)); + doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pRuntimeEnv->resultInfo.capacity); + return (pInfo->pRes->info.rows > 0)? pInfo->pRes:NULL; } +} - for (int32_t i = 0; i < pQuery->numOfExpr2; ++i) { - tfree(data[i]); +// todo set the attribute of query scan count +static int32_t getNumOfScanTimes(SQuery* pQuery) { + for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functionId = pQuery->pExpr1[i].base.functionId; + if (functionId == TSDB_FUNC_STDDEV || functionId == TSDB_FUNC_PERCT) { + return 2; + } } - tfree(data); - tfree(arithSup.data); + return 1; } -/* - * in each query, this function will be called only once, no retry for further result. - * - * select count(*)/top(field,k)/avg(field name) from table_name [where ts>now-1a]; - * select count(*) from table_name group by status_column; - */ -static void tableAggregationProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - - SQuery *pQuery = pRuntimeEnv->pQuery; - if (!pRuntimeEnv->topBotQuery && pQuery->limit.offset > 0) { // no need to execute, since the output will be ignore. +static void destroyOperatorInfo(SOperatorInfo* pOperator) { + if (pOperator == NULL) { return; } - scanOneTableDataBlocks(pRuntimeEnv, pTableInfo->lastKey); - finalizeQueryResult(pRuntimeEnv); + if (pOperator->cleanup != NULL) { + pOperator->cleanup(pOperator->info, pOperator->numOfOutput); + } - // since the numOfRows must be identical for all sql functions that are allowed to be executed simutaneously. - pQuery->rec.rows = getNumOfResult(pRuntimeEnv); - doSecondaryArithmeticProcess(pQuery); + destroyOperatorInfo(pOperator->upstream); + tfree(pOperator->info); + tfree(pOperator); +} - if (isQueryKilled(pQInfo)) { - longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); - } +static SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); - // TODO limit/offset refactor to be one operator - skipResults(pRuntimeEnv); - limitOperator(pQuery, pQInfo); + SQuery* pQuery = pRuntimeEnv->pQuery; + int32_t numOfRows = (int32_t)(GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pQuery->topBotQuery, pQuery->stableQuery)); + + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, numOfRows); + pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + pInfo->seed = rand(); + setDefaultOutputBuf(pRuntimeEnv, &pInfo->binfo, pInfo->seed); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "TableAggregate"; + pOperator->operatorType = OP_Aggregate; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->upstream = upstream; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = pRuntimeEnv; + + pOperator->exec = doAggregate; + pOperator->cleanup = destroyBasicOperatorInfo; + return pOperator; } -static void tableProjectionProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; +static void doDestroyBasicInfo(SOptrBasicInfo* pInfo, int32_t numOfOutput) { + assert(pInfo != NULL); - // for ts_comp query, re-initialized is not allowed - SQuery *pQuery = pRuntimeEnv->pQuery; - if (!isTsCompQuery(pQuery)) { - resetDefaultResInfoOutputBuf(pRuntimeEnv); - } + destroySQLFunctionCtx(pInfo->pCtx, numOfOutput); + tfree(pInfo->rowCellInfoOffset); - // skip blocks without load the actual data block from file if no filter condition present - skipBlocks(&pQInfo->runtimeEnv); - if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0) { - setQueryStatus(pQuery, QUERY_COMPLETED); - return; - } + cleanupResultRowInfo(&pInfo->resultRowInfo); + pInfo->pRes = destroyOutputBuf(pInfo->pRes); +} - while (1) { - scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey); - finalizeQueryResult(pRuntimeEnv); +static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) { + SOptrBasicInfo* pInfo = (SOptrBasicInfo*) param; + doDestroyBasicInfo(pInfo, numOfOutput); +} - pQuery->rec.rows = getNumOfResult(pRuntimeEnv); - if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols > 0 && pQuery->rec.rows > 0) { - skipResults(pRuntimeEnv); - } +static void destroySFillOperatorInfo(void* param, int32_t numOfOutput) { + SFillOperatorInfo* pInfo = (SFillOperatorInfo*) param; + pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo); + pInfo->pRes = destroyOutputBuf(pInfo->pRes); +} - /* - * 1. if pQuery->size == 0, pQuery->limit.offset >= 0, still need to check data - * 2. if pQuery->size > 0, pQuery->limit.offset must be 0 - */ - if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - break; - } +static void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput) { + SGroupbyOperatorInfo* pInfo = (SGroupbyOperatorInfo*) param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); + tfree(pInfo->prevData); +} + +static void destroyArithOperatorInfo(void* param, int32_t numOfOutput) { + SArithOperatorInfo* pInfo = (SArithOperatorInfo*) param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); +} - qDebug("QInfo:%p skip current result, offset:%" PRId64 ", next qrange:%" PRId64 "-%" PRId64, - pQInfo, pQuery->limit.offset, pQuery->current->lastKey, pQuery->current->win.ekey); +static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { + STagScanInfo* pInfo = (STagScanInfo*) param; + pInfo->pRes = destroyOutputBuf(pInfo->pRes); +} - resetDefaultResInfoOutputBuf(pRuntimeEnv); - } +SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); - limitOperator(pQuery, pQInfo); - if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { - qDebug("QInfo:%p query paused due to output limitation, next qrange:%" PRId64 "-%" PRId64, pQInfo, - pQuery->current->lastKey, pQuery->window.ekey); - } else if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - STableIdInfo tidInfo = createTableIdInfo(pQuery); - taosHashPut(pQInfo->arrTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid), &tidInfo, sizeof(STableIdInfo)); - } + size_t tableGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); - if (!isTsCompQuery(pQuery)) { - assert(pQuery->rec.rows <= pQuery->rec.capacity); - } + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, (int32_t) tableGroup); + pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)tableGroup, TSDB_DATA_TYPE_INT); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "MultiTableAggregate"; + pOperator->operatorType = OP_MultiTableAggregate; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->upstream = upstream; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = pRuntimeEnv; + + pOperator->exec = doSTableAggregate; + pOperator->cleanup = destroyBasicOperatorInfo; + + return pOperator; } -static void copyAndFillResult(SQInfo* pQInfo) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery* pQuery = pRuntimeEnv->pQuery; +SOperatorInfo* createArithOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + SArithOperatorInfo* pInfo = calloc(1, sizeof(SArithOperatorInfo)); - while(1) { - copyToOutputBuf(pQInfo, &pRuntimeEnv->resultRowInfo); - doSecondaryArithmeticProcess(pQuery); + pInfo->seed = rand(); + pInfo->bufCapacity = pRuntimeEnv->resultInfo.capacity; - TSKEY lastKey = 0; - if (!hasRemainData(&pQInfo->groupResInfo)) { - lastKey = pQuery->window.ekey; - } else { - lastKey = ((TSKEY*)pQuery->sdata[0]->data)[pQuery->rec.rows - 1]; - } + SOptrBasicInfo* pBInfo = &pInfo->binfo; + pBInfo->pRes = createOutputBuf(pExpr, numOfOutput, pInfo->bufCapacity); + pBInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pBInfo->rowCellInfoOffset); - assert(lastKey <= pQuery->window.ekey); + initResultRowInfo(&pBInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + setDefaultOutputBuf(pRuntimeEnv, pBInfo, pInfo->seed); - taosFillSetStartInfo(pRuntimeEnv->pFillInfo, (int32_t)pQuery->rec.rows, lastKey); - taosFillSetDataBlockFromFilePage(pRuntimeEnv->pFillInfo, (const tFilePage **)pQuery->sdata); + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "ArithmeticOperator"; + pOperator->operatorType = OP_Arithmetic; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->upstream = upstream; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = pRuntimeEnv; - pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata); + pOperator->exec = doArithmeticOperation; + pOperator->cleanup = destroyArithOperatorInfo; - if (pQuery->rec.rows > 0) { - limitOperator(pQuery, pQInfo); - break; - } + return pOperator; +} - // here the pQuery->rec.rows == 0 - if (!hasRemainData(&pQInfo->groupResInfo) && !taosFillHasMoreResults(pRuntimeEnv->pFillInfo)) { - break; - } - } +SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream) { + SLimitOperatorInfo* pInfo = calloc(1, sizeof(SLimitOperatorInfo)); + pInfo->limit = pRuntimeEnv->pQuery->limit.limit; + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + + pOperator->name = "LimitOperator"; + pOperator->operatorType = OP_Limit; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->upstream = upstream; + pOperator->exec = doLimit; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + + return pOperator; } -// handle time interval query on table -static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &(pQInfo->runtimeEnv); - SQuery *pQuery = pRuntimeEnv->pQuery; +SOperatorInfo* createOffsetOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream) { + SOffsetOperatorInfo* pInfo = calloc(1, sizeof(SOffsetOperatorInfo)); - TSKEY newStartKey = QUERY_IS_ASC_QUERY(pQuery)? INT64_MIN:INT64_MAX; + pInfo->offset = pRuntimeEnv->pQuery->limit.offset; + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - // skip blocks without load the actual data block from file if no filter condition present - if (!pRuntimeEnv->groupbyColumn) { - skipTimeInterval(pRuntimeEnv, &newStartKey); - if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0 && pRuntimeEnv->pFillInfo == NULL) { - setQueryStatus(pQuery, QUERY_COMPLETED); - return; - } - } + pOperator->name = "OffsetOperator"; + pOperator->operatorType = OP_Offset; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->upstream = upstream; + pOperator->exec = doOffset; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; - scanOneTableDataBlocks(pRuntimeEnv, newStartKey); - finalizeQueryResult(pRuntimeEnv); + return pOperator; +} - // skip offset result rows - pQuery->rec.rows = 0; +SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - // not fill or no result generated during this query - if (pQuery->fillType == TSDB_FILL_NONE || pRuntimeEnv->resultRowInfo.size == 0 || isPointInterpoQuery(pQuery)) { - // all data scanned, the group by normal column can return - int32_t numOfClosed = numOfClosedResultRows(&pRuntimeEnv->resultRowInfo); - if (pQuery->limit.offset > numOfClosed || numOfClosed == 0) { - return; - } + pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); - initGroupResInfo(&pQInfo->groupResInfo, &pRuntimeEnv->resultRowInfo, (int32_t) pQuery->limit.offset); - copyToOutputBuf(pQInfo, &pRuntimeEnv->resultRowInfo); - doSecondaryArithmeticProcess(pQuery); + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - limitOperator(pQuery, pQInfo); - } else { - initGroupResInfo(&pQInfo->groupResInfo, &pRuntimeEnv->resultRowInfo, 0); - copyAndFillResult(pQInfo); + pOperator->name = "TimeIntervalAggOperator"; + pOperator->operatorType = OP_TimeWindow; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->upstream = upstream; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = doIntervalAgg; + pOperator->cleanup = destroyBasicOperatorInfo; + + return pOperator; +} + +SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + SSWindowOperatorInfo* pInfo = calloc(1, sizeof(SSWindowOperatorInfo)); + + pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + pInfo->prevTs = INT64_MIN; + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + + pOperator->name = "SessionWindowAggOperator"; + pOperator->operatorType = OP_SessionWindow; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->upstream = upstream; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = doSessionWindowAgg; + pOperator->cleanup = destroyBasicOperatorInfo; + + return pOperator; +} + +SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); + + pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "MultiTableTimeIntervalOperator"; + pOperator->operatorType = OP_MultiTableTimeInterval; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->upstream = upstream; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + + pOperator->exec = doSTableIntervalAgg; + pOperator->cleanup = destroyBasicOperatorInfo; + + return pOperator; +} + +SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + SGroupbyOperatorInfo* pInfo = calloc(1, sizeof(SGroupbyOperatorInfo)); + pInfo->colIndex = -1; // group by column index + + pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "GroupbyAggOperator"; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->operatorType = OP_Groupby; + pOperator->upstream = upstream; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = hashGroupbyAggregate; + pOperator->cleanup = destroyGroupbyOperatorInfo; + + return pOperator; +} + +SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, + int32_t numOfOutput) { + SFillOperatorInfo* pInfo = calloc(1, sizeof(SFillOperatorInfo)); + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + + { + SQuery* pQuery = pRuntimeEnv->pQuery; + SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfOutput, pQuery->fillVal); + STimeWindow w = TSWINDOW_INITIALIZER; + + TSKEY sk = MIN(pQuery->window.skey, pQuery->window.ekey); + TSKEY ek = MAX(pQuery->window.skey, pQuery->window.ekey); + getAlignQueryTimeWindow(pQuery, pQuery->window.skey, sk, ek, &w); + + pInfo->pFillInfo = taosCreateFillInfo(pQuery->order.order, w.skey, 0, (int32_t)pRuntimeEnv->resultInfo.capacity, numOfOutput, + pQuery->interval.sliding, pQuery->interval.slidingUnit, (int8_t)pQuery->precision, + pQuery->fillType, pColInfo, pRuntimeEnv->qinfo); } + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + + pOperator->name = "FillOperator"; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->operatorType = OP_Fill; + + pOperator->upstream = upstream; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + + pOperator->exec = doFill; + pOperator->cleanup = destroySFillOperatorInfo; + + return pOperator; } -void tableQueryImpl(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; +static SSDataBlock* doTagScan(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } - if (hasNotReturnedResults(pRuntimeEnv, &pQInfo->groupResInfo)) { - if (pQuery->fillType != TSDB_FILL_NONE && !isPointInterpoQuery(pQuery)) { - /* - * There are remain results that are not returned due to result interpolation - * So, we do keep in this procedure instead of launching retrieve procedure for next results. - */ - pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata); - if (pQuery->rec.rows > 0) { - limitOperator(pQuery, pQInfo); - qDebug("QInfo:%p current:%" PRId64 " returned, total:%" PRId64, pQInfo, pQuery->rec.rows, pQuery->rec.total); - } else { - copyAndFillResult(pQInfo); + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + + int32_t maxNumOfTables = (int32_t)pRuntimeEnv->resultInfo.capacity; + + STagScanInfo *pInfo = pOperator->info; + SSDataBlock *pRes = pInfo->pRes; + + int32_t count = 0; + SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0); + + int32_t functionId = pOperator->pExpr[0].base.functionId; + if (functionId == TSDB_FUNC_TID_TAG) { // return the tags & table Id + SQuery* pQuery = pRuntimeEnv->pQuery; + assert(pQuery->numOfOutput == 1); + + SExprInfo* pExprInfo = &pOperator->pExpr[0]; + int32_t rsize = pExprInfo->bytes; + + count = 0; + + int16_t bytes = pExprInfo->bytes; + int16_t type = pExprInfo->type; + + for(int32_t i = 0; i < pQuery->numOfTags; ++i) { + if (pQuery->tagColList[i].colId == pExprInfo->base.colInfo.colId) { + bytes = pQuery->tagColList[i].bytes; + type = pQuery->tagColList[i].type; + break; } + } + + SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); + + while(pInfo->currentIndex < pInfo->totalTables && count < maxNumOfTables) { + int32_t i = pInfo->currentIndex++; + STableQueryInfo *item = taosArrayGetP(pa, i); + + char *output = pColInfo->pData + count * rsize; + varDataSetLen(output, rsize - VARSTR_HEADER_SIZE); + + output = varDataVal(output); + STableId* id = TSDB_TABLEID(item->pTable); + + *(int16_t *)output = 0; + output += sizeof(int16_t); + + *(int64_t *)output = id->uid; // memory align problem, todo serialize + output += sizeof(id->uid); - } else { - pQuery->rec.rows = 0; - assert(pRuntimeEnv->resultRowInfo.size > 0); - copyToOutputBuf(pQInfo, &pRuntimeEnv->resultRowInfo); - doSecondaryArithmeticProcess(pQuery); + *(int32_t *)output = id->tid; + output += sizeof(id->tid); - if (pQuery->rec.rows > 0) { - limitOperator(pQuery, pQInfo); - } + *(int32_t *)output = pQuery->vgId; + output += sizeof(pQuery->vgId); - if (pQuery->rec.rows > 0) { - qDebug("QInfo:%p %" PRId64 " rows returned from group results, total:%" PRId64 "", pQInfo, pQuery->rec.rows, - pQuery->rec.total); + char* data = NULL; + if (pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { + data = tsdbGetTableName(item->pTable); } else { - qDebug("QInfo:%p query over, %" PRId64 " rows are returned", pQInfo, pQuery->rec.total); + data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.colInfo.colId, type, bytes); } - } - return; - } + doSetTagValueToResultBuf(output, data, type, bytes); + count += 1; + } - // number of points returned during this query - pQuery->rec.rows = 0; - int64_t st = taosGetTimestampUs(); + qDebug("QInfo:%p create (tableId, tag) info completed, rows:%d", pRuntimeEnv->qinfo, count); + } else if (functionId == TSDB_FUNC_COUNT) {// handle the "count(tbname)" query + SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); + *(int64_t*)pColInfo->pData = pInfo->totalTables; + count = 1; - assert(pQInfo->tableqinfoGroupInfo.numOfTables == 1); - SArray* g = GET_TABLEGROUP(pQInfo, 0); + pOperator->status = OP_EXEC_DONE; + qDebug("QInfo:%p create count(tbname) query, res:%d rows:1", pRuntimeEnv->qinfo, count); + } else { // return only the tags|table name etc. + SExprInfo* pExprInfo = pOperator->pExpr; // todo use the column list instead of exprinfo - STableQueryInfo* item = taosArrayGetP(g, 0); - pQuery->current = item; + count = 0; + while(pInfo->currentIndex < pInfo->totalTables && count < maxNumOfTables) { + int32_t i = pInfo->currentIndex++; - // group by normal column, sliding window query, interval query are handled by interval query processor - if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyColumn) { // interval (down sampling operation) - tableIntervalProcess(pQInfo, item); - } else if (isFixedOutputQuery(pRuntimeEnv)) { - tableAggregationProcess(pQInfo, item); - } else { // diff/add/multiply/subtract/division - assert(pQuery->checkResultBuf == 1); - tableProjectionProcess(pQInfo, item); - } + STableQueryInfo* item = taosArrayGetP(pa, i); - // record the total elapsed time - pRuntimeEnv->summary.elapsedTime += (taosGetTimestampUs() - st); - assert(pQInfo->tableqinfoGroupInfo.numOfTables == 1); -} + char *data = NULL, *dst = NULL; + int16_t type = 0, bytes = 0; + for(int32_t j = 0; j < pOperator->numOfOutput; ++j) { + // not assign value in case of user defined constant output column + if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.colInfo.flag)) { + continue; + } -void buildTableBlockDistResult(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery *pQuery = pRuntimeEnv->pQuery; - pQuery->pos = 0; + SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, j); + type = pExprInfo[j].type; + bytes = pExprInfo[j].bytes; - STableBlockDist *pTableBlockDist = calloc(1, sizeof(STableBlockDist)); - pTableBlockDist->dataBlockInfos = taosArrayInit(512, sizeof(SDataBlockInfo)); - pTableBlockDist->result = (char *)malloc(512); + if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { + data = tsdbGetTableName(item->pTable); + } else { + data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes); + } - TsdbQueryHandleT pQueryHandle = pRuntimeEnv->pQueryHandle; - SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; - SSchema blockDistSchema = tGetBlockDistColumnSchema(); + dst = pColInfo->pData + count * pExprInfo[j].bytes; + doSetTagValueToResultBuf(dst, data, type, bytes); + } - int64_t startTime = taosGetTimestampUs(); - while (tsdbNextDataBlockWithoutMerge(pQueryHandle)) { - if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { - freeTableBlockDist(pTableBlockDist); - longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); + count += 1; } - if (pTableBlockDist->firstSeekTimeUs == 0) { - pTableBlockDist->firstSeekTimeUs = taosGetTimestampUs() - startTime; - } - - tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); - taosArrayPush(pTableBlockDist->dataBlockInfos, &blockInfo); - } - if (terrno != TSDB_CODE_SUCCESS) { - freeTableBlockDist(pTableBlockDist); - longjmp(pRuntimeEnv->env, terrno); - } - pTableBlockDist->numOfRowsInMemTable = tsdbGetNumOfRowsInMemTable(pQueryHandle); - - generateBlockDistResult(pTableBlockDist); - - int type = -1; - assert(pQuery->numOfOutput == 1); - SExprInfo* pExprInfo = pQuery->pExpr1; - for (int32_t j = 0; j < pQuery->numOfOutput; j++) { - if (pExprInfo[j].base.colInfo.colId == TSDB_BLOCK_DIST_COLUMN_INDEX) { - type = blockDistSchema.type; + if (pInfo->currentIndex >= pInfo->totalTables) { + pOperator->status = OP_EXEC_DONE; } - assert(type == TSDB_DATA_TYPE_BINARY); - STR_WITH_SIZE_TO_VARSTR(pQuery->sdata[j]->data, pTableBlockDist->result, (VarDataLenT)strlen(pTableBlockDist->result)); - } - freeTableBlockDist(pTableBlockDist); + qDebug("QInfo:%p create tag values results completed, rows:%d", pRuntimeEnv->qinfo, count); + } - pQuery->rec.rows = 1; - setQueryStatus(pQuery, QUERY_COMPLETED); - return; + pRes->info.rows = count; + return (pRes->info.rows == 0)? NULL:pInfo->pRes; } -void stableQueryImpl(SQInfo *pQInfo) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery *pQuery = pRuntimeEnv->pQuery; - pQuery->rec.rows = 0; +SOperatorInfo* createTagScanOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput) { + STagScanInfo* pInfo = calloc(1, sizeof(STagScanInfo)); + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - int64_t st = taosGetTimestampUs(); + size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); + assert(numOfGroup == 0 || numOfGroup == 1); - if (QUERY_IS_INTERVAL_QUERY(pQuery) || - (isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && (!pRuntimeEnv->groupbyColumn))) { - multiTableQueryProcess(pQInfo); - } else { - assert(pQuery->checkResultBuf == 1 || isPointInterpoQuery(pQuery) || pRuntimeEnv->groupbyColumn); - sequentialTableProcess(pQInfo); - } + pInfo->totalTables = pRuntimeEnv->tableqinfoGroupInfo.numOfTables; + pInfo->currentIndex = 0; - // record the total elapsed time - pQInfo->runtimeEnv.summary.elapsedTime += (taosGetTimestampUs() - st); + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "SeqTableTagScan"; + pOperator->operatorType = OP_TagScan; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->exec = doTagScan; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->cleanup = destroyTagScanOperatorInfo; + + return pOperator; } static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pExprMsg, SColumnInfo* pTagCols) { @@ -5934,6 +5357,17 @@ static bool validateQueryMsg(SQueryTableMsg *pQueryMsg) { return false; } + if (pQueryMsg->sw.gap < 0 || pQueryMsg->sw.primaryColId != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + qError("qmsg:%p illegal value of session window time %" PRId64, pQueryMsg, pQueryMsg->sw.gap); + return false; + } + + if (pQueryMsg->sw.gap > 0 && pQueryMsg->interval.interval > 0) { + qError("qmsg:%p illegal value of session window time %" PRId64" and interval value %"PRId64, pQueryMsg, + pQueryMsg->sw.gap, pQueryMsg->interval.interval); + return false; + } + if (pQueryMsg->numOfTables <= 0) { qError("qmsg:%p illegal value of numOfTables %d", pQueryMsg, pQueryMsg->numOfTables); return false; @@ -5965,7 +5399,8 @@ static bool validateQuerySourceCols(SQueryTableMsg *pQueryMsg, SSqlFuncMsg** pEx if ((pFuncMsg->functionId == TSDB_FUNC_TAGPRJ) || (pFuncMsg->functionId == TSDB_FUNC_TID_TAG && pFuncMsg->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) || - (pFuncMsg->functionId == TSDB_FUNC_COUNT && pFuncMsg->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX)) { + (pFuncMsg->functionId == TSDB_FUNC_COUNT && pFuncMsg->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) || + (pFuncMsg->functionId == TSDB_FUNC_BLKINFO)) { continue; } @@ -6045,6 +5480,8 @@ int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) { pQueryMsg->secondStageOutput = htonl(pQueryMsg->secondStageOutput); pQueryMsg->sqlstrLen = htonl(pQueryMsg->sqlstrLen); pQueryMsg->prevResultLen = htonl(pQueryMsg->prevResultLen); + pQueryMsg->sw.gap = htobe64(pQueryMsg->sw.gap); + pQueryMsg->sw.primaryColId = htonl(pQueryMsg->sw.primaryColId); // query msg safety check if (!validateQueryMsg(pQueryMsg)) { @@ -6119,6 +5556,9 @@ int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) { pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); + pExprMsg->colType = htons(pExprMsg->colType); + pExprMsg->colBytes = htons(pExprMsg->colBytes); + pExprMsg->functionId = htons(pExprMsg->functionId); pExprMsg->numOfParams = htons(pExprMsg->numOfParams); pExprMsg->resColId = htons(pExprMsg->resColId); @@ -6157,7 +5597,10 @@ int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) { pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); - pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); + pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); + pExprMsg->colType = htons(pExprMsg->colType); + pExprMsg->colBytes = htons(pExprMsg->colBytes); + pExprMsg->functionId = htons(pExprMsg->functionId); pExprMsg->numOfParams = htons(pExprMsg->numOfParams); @@ -6327,8 +5770,29 @@ static int32_t buildArithmeticExprFromMsg(SExprInfo *pArithExprInfo, SQueryTable return TSDB_CODE_SUCCESS; } -int32_t createQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo, SSqlFuncMsg **pExprMsg, - SColumnInfo* pTagCols) { +static int32_t updateOutputBufForTopBotQuery(SQueryTableMsg* pQueryMsg, SColumnInfo* pTagCols, SExprInfo* pExprs, int32_t numOfOutput, int32_t tagLen, bool superTable) { + for (int32_t i = 0; i < numOfOutput; ++i) { + int16_t functId = pExprs[i].base.functionId; + + if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { + int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); + if (j < 0 || j >= pQueryMsg->numOfCols) { + return TSDB_CODE_QRY_INVALID_MSG; + } else { + SColumnInfo* pCol = &pQueryMsg->colList[j]; + int32_t ret = getResultDataInfo(pCol->type, pCol->bytes, functId, (int32_t)pExprs[i].base.arg[0].argValue.i64, + &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interBytes, tagLen, superTable); + assert(ret == TSDB_CODE_SUCCESS); + } + } + } + + return TSDB_CODE_SUCCESS; +} + +// TODO tag length should be passed from client +int32_t createQueryFuncExprFromMsg(SQueryTableMsg* pQueryMsg, int32_t numOfOutput, SExprInfo** pExprInfo, + SSqlFuncMsg** pExprMsg, SColumnInfo* pTagCols) { *pExprInfo = NULL; int32_t code = TSDB_CODE_SUCCESS; @@ -6401,6 +5865,12 @@ int32_t createQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutpu } int32_t param = (int32_t)pExprs[i].base.arg[0].argValue.i64; + if (pExprs[i].base.functionId != TSDB_FUNC_ARITHM && + (type != pExprs[i].base.colType || bytes != pExprs[i].base.colBytes)) { + tfree(pExprs); + return TSDB_CODE_QRY_INVALID_MSG; + } + if (getResultDataInfo(type, bytes, pExprs[i].base.functionId, param, &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interBytes, 0, isSuperTable) != TSDB_CODE_SUCCESS) { tfree(pExprs); @@ -6414,23 +5884,59 @@ int32_t createQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutpu assert(isValidDataType(pExprs[i].type)); } - // TODO refactor + // the tag length is affected by other tag columns, so this should be update. + updateOutputBufForTopBotQuery(pQueryMsg, pTagCols, pExprs, numOfOutput, tagLen, isSuperTable); + + *pExprInfo = pExprs; + return TSDB_CODE_SUCCESS; +} + +int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo, + SSqlFuncMsg **pExprMsg, SExprInfo *prevExpr) { + *pExprInfo = NULL; + int32_t code = TSDB_CODE_SUCCESS; + + SExprInfo *pExprs = (SExprInfo *)calloc(numOfOutput, sizeof(SExprInfo)); + if (pExprs == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + bool isSuperTable = QUERY_IS_STABLE_QUERY(pQueryMsg->queryType); + for (int32_t i = 0; i < numOfOutput; ++i) { pExprs[i].base = *pExprMsg[i]; - int16_t functId = pExprs[i].base.functionId; + pExprs[i].bytes = 0; - if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { - int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); - if (j < 0 || j >= pQueryMsg->numOfCols) { - return TSDB_CODE_QRY_INVALID_MSG; - } else { - SColumnInfo *pCol = &pQueryMsg->colList[j]; - int32_t ret = - getResultDataInfo(pCol->type, pCol->bytes, functId, (int32_t)pExprs[i].base.arg[0].argValue.i64, - &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interBytes, tagLen, isSuperTable); - assert(ret == TSDB_CODE_SUCCESS); + int16_t type = 0; + int16_t bytes = 0; + + // parse the arithmetic expression + if (pExprs[i].base.functionId == TSDB_FUNC_ARITHM) { + code = buildArithmeticExprFromMsg(&pExprs[i], pQueryMsg); + + if (code != TSDB_CODE_SUCCESS) { + tfree(pExprs); + return code; } + + type = TSDB_DATA_TYPE_DOUBLE; + bytes = tDataTypes[type].bytes; + } else { + int32_t index = pExprs[i].base.colInfo.colIndex; + assert(prevExpr[index].base.resColId == pExprs[i].base.colInfo.colId); + + type = prevExpr[index].type; + bytes = prevExpr[index].bytes; + } + + int32_t param = (int32_t)pExprs[i].base.arg[0].argValue.i64; + if (getResultDataInfo(type, bytes, pExprs[i].base.functionId, param, &pExprs[i].type, &pExprs[i].bytes, + &pExprs[i].interBytes, 0, isSuperTable) != TSDB_CODE_SUCCESS) { + tfree(pExprs); + return TSDB_CODE_QRY_INVALID_MSG; } + + assert(isValidDataType(pExprs[i].type)); } *pExprInfo = pExprs; @@ -6540,6 +6046,9 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { assert(f < pQuery->numOfCols); } else if (pColIndex->colId <= TSDB_UD_COLUMN_INDEX) { // do nothing for user-defined constant value result columns + } else if (pColIndex->colId == TSDB_BLOCK_DIST_COLUMN_INDEX) { + pColIndex->colIndex = 0;// only one source column, so it must be 0; + assert(pQuery->numOfOutput == 1); } else { int32_t f = 0; for (f = 0; f < pQuery->numOfTags; ++f) { @@ -6554,31 +6063,37 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { } } -static void calResultBufSize(SQuery* pQuery) { - const int32_t RESULT_MSG_MIN_SIZE = 1024 * (1024 + 512); // bytes - const int32_t RESULT_MSG_MIN_ROWS = 8192; - const float RESULT_THRESHOLD_RATIO = 0.85f; +void setResultBufSize(SQuery* pQuery, SRspResultInfo* pResultInfo) { + const int32_t DEFAULT_RESULT_MSG_SIZE = 1024 * (1024 + 512); + + // the minimum number of rows for projection query + const int32_t MIN_ROWS_FOR_PRJ_QUERY = 8192; + const int32_t DEFAULT_MIN_ROWS = 4096; + + const float THRESHOLD_RATIO = 0.85f; if (isProjQuery(pQuery)) { - int32_t numOfRes = RESULT_MSG_MIN_SIZE / pQuery->resultRowSize; - if (numOfRes < RESULT_MSG_MIN_ROWS) { - numOfRes = RESULT_MSG_MIN_ROWS; + int32_t numOfRes = DEFAULT_RESULT_MSG_SIZE / pQuery->resultRowSize; + if (numOfRes < MIN_ROWS_FOR_PRJ_QUERY) { + numOfRes = MIN_ROWS_FOR_PRJ_QUERY; } - pQuery->rec.capacity = numOfRes; - pQuery->rec.threshold = (int32_t)(numOfRes * RESULT_THRESHOLD_RATIO); + pResultInfo->capacity = numOfRes; } else { // in case of non-prj query, a smaller output buffer will be used. - pQuery->rec.capacity = 4096; - pQuery->rec.threshold = (int32_t)(pQuery->rec.capacity * RESULT_THRESHOLD_RATIO); + pResultInfo->capacity = DEFAULT_MIN_ROWS; } + + pResultInfo->threshold = (int32_t)(pResultInfo->capacity * THRESHOLD_RATIO); + pResultInfo->total = 0; } FORCE_INLINE bool checkQIdEqual(void *qHandle, uint64_t qId) { return ((SQInfo *)qHandle)->qId == qId; } -SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, - SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, bool stableQuery, char* sql, uint64_t *qId) { +SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SSqlGroupbyExpr* pGroupbyExpr, SExprInfo* pExprs, + SExprInfo* pSecExprs, STableGroupInfo* pTableGroupInfo, SColumnInfo* pTagCols, bool stableQuery, + char* sql) { int16_t numOfCols = pQueryMsg->numOfCols; int16_t numOfOutput = pQueryMsg->numOfOutput; @@ -6589,15 +6104,10 @@ SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr // to make sure third party won't overwrite this structure pQInfo->signature = pQInfo; - pQInfo->tableGroupInfo = *pTableGroupInfo; - - SQuery *pQuery = calloc(1, sizeof(SQuery)); - if (pQuery == NULL) { - goto _cleanup_query; - } - + SQuery* pQuery = &pQInfo->query; pQInfo->runtimeEnv.pQuery = pQuery; + pQuery->tableGroupInfo = *pTableGroupInfo; pQuery->numOfCols = numOfCols; pQuery->numOfOutput = numOfOutput; pQuery->limit.limit = pQueryMsg->limit; @@ -6614,7 +6124,7 @@ SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr pQuery->tagColList = pTagCols; pQuery->prjInfo.vgroupLimit = pQueryMsg->vgroupLimit; pQuery->prjInfo.ts = (pQueryMsg->order == TSDB_ORDER_ASC)? INT64_MIN:INT64_MAX; - + pQuery->sw = pQueryMsg->sw; pQuery->colList = calloc(numOfCols, sizeof(SSingleColumnFilterInfo)); if (pQuery->colList == NULL) { goto _cleanup; @@ -6644,37 +6154,11 @@ SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr } doUpdateExprColumnIndex(pQuery); - int32_t ret = createFilterInfo(pQInfo, pQuery); if (ret != TSDB_CODE_SUCCESS) { goto _cleanup; } - // prepare the result buffer - pQuery->sdata = (tFilePage **)calloc(pQuery->numOfOutput, POINTER_BYTES); - if (pQuery->sdata == NULL) { - goto _cleanup; - } - - calResultBufSize(pQuery); - - for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { - // allocate additional memory for interResults that are usually larger then final results - // TODO refactor - int16_t bytes = 0; - if (pQuery->pExpr2 == NULL || col > pQuery->numOfExpr2) { - bytes = pExprs[col].bytes; - } else { - bytes = MAX(pQuery->pExpr2[col].bytes, pExprs[col].bytes); - } - - size_t size = (size_t)((pQuery->rec.capacity + 1) * bytes + pExprs[col].interBytes + sizeof(tFilePage)); - pQuery->sdata[col] = (tFilePage *)calloc(1, size); - if (pQuery->sdata[col] == NULL) { - goto _cleanup; - } - } - if (pQuery->fillType != TSDB_FILL_NONE) { pQuery->fillVal = malloc(sizeof(int64_t) * pQuery->numOfOutput); if (pQuery->fillVal == NULL) { @@ -6688,7 +6172,7 @@ SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr size_t numOfGroups = 0; if (pTableGroupInfo->pGroupList != NULL) { numOfGroups = taosArrayGetSize(pTableGroupInfo->pGroupList); - STableGroupInfo* pTableqinfo = &pQInfo->tableqinfoGroupInfo; + STableGroupInfo* pTableqinfo = &pQInfo->runtimeEnv.tableqinfoGroupInfo; pTableqinfo->pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES); pTableqinfo->numOfTables = pTableGroupInfo->numOfTables; @@ -6700,26 +6184,23 @@ SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr goto _cleanup; } - // NOTE: pTableCheckInfo need to update the query time range and the lastKey info - pQInfo->arrTableIdInfo = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); pQInfo->dataReady = QUERY_RESULT_NOT_READY; pQInfo->rspContext = NULL; pQInfo->sql = sql; pthread_mutex_init(&pQInfo->lock, NULL); tsem_init(&pQInfo->ready, 0, 0); - pQuery->pos = -1; pQuery->window = pQueryMsg->window; changeExecuteScanOrder(pQInfo, pQueryMsg, stableQuery); - pQInfo->runtimeEnv.queryWindowIdentical = true; + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; bool groupByCol = isGroupbyColumn(pQuery->pGroupbyExpr); STimeWindow window = pQuery->window; int32_t index = 0; for(int32_t i = 0; i < numOfGroups; ++i) { - SArray* pa = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, i); + SArray* pa = taosArrayGetP(pQuery->tableGroupInfo.pGroupList, i); size_t s = taosArrayGetSize(pa); SArray* p1 = taosArrayInit(s, POINTER_BYTES); @@ -6727,15 +6208,11 @@ SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr goto _cleanup; } - taosArrayPush(pQInfo->tableqinfoGroupInfo.pGroupList, &p1); + taosArrayPush(pRuntimeEnv->tableqinfoGroupInfo.pGroupList, &p1); for(int32_t j = 0; j < s; ++j) { STableKeyInfo* info = taosArrayGet(pa, j); - window.skey = info->lastKey; - if (info->lastKey != pQuery->window.skey) { - pQInfo->runtimeEnv.queryWindowIdentical = false; - } void* buf = (char*) pQInfo->pBuf + index * sizeof(STableQueryInfo); STableQueryInfo* item = createTableQueryInfo(pQuery, info->pTable, groupByCol, window, buf); @@ -6747,15 +6224,15 @@ SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr taosArrayPush(p1, &item); STableId* id = TSDB_TABLEID(info->pTable); - taosHashPut(pQInfo->tableqinfoGroupInfo.map, &id->tid, sizeof(id->tid), &item, POINTER_BYTES); + taosHashPut(pRuntimeEnv->tableqinfoGroupInfo.map, &id->tid, sizeof(id->tid), &item, POINTER_BYTES); index += 1; } } - colIdCheck(pQuery); + colIdCheck(pQuery, pQInfo); // todo refactor - pQInfo->runtimeEnv.queryBlockDist = (numOfOutput == 1 && pExprs[0].base.colInfo.colId == TSDB_BLOCK_DIST_COLUMN_INDEX); + pQInfo->query.queryBlockDist = (numOfOutput == 1 && pExprs[0].base.colInfo.colId == TSDB_BLOCK_DIST_COLUMN_INDEX); pQInfo->qId = atomic_add_fetch_64(&queryHandleId, 1); *qId = pQInfo->qId; @@ -6765,7 +6242,6 @@ SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr _cleanup_qinfo: tsdbDestroyTableGroup(pTableGroupInfo); -_cleanup_query: if (pGroupbyExpr != NULL) { taosArrayDestroy(pGroupbyExpr->columnInfo); free(pGroupbyExpr); @@ -6803,6 +6279,8 @@ bool isValidQInfo(void *param) { int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *pQInfo, SQueryParam* param, bool isSTable) { int32_t code = TSDB_CODE_SUCCESS; + + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; STSBuf *pTsBuf = NULL; @@ -6827,15 +6305,15 @@ int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *p (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.ekey > pQuery->window.skey))) { qDebug("QInfo:%p no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo, pQuery->window.skey, pQuery->window.ekey, pQuery->order.order); - setQueryStatus(pQuery, QUERY_COMPLETED); - pQInfo->tableqinfoGroupInfo.numOfTables = 0; + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + pRuntimeEnv->tableqinfoGroupInfo.numOfTables = 0; // todo free memory return TSDB_CODE_SUCCESS; } - if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) { + if (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 0) { qDebug("QInfo:%p no table qualified for tag filter, abort query", pQInfo); - setQueryStatus(pQuery, QUERY_COMPLETED); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); return TSDB_CODE_SUCCESS; } @@ -6913,18 +6391,12 @@ void freeQInfo(SQInfo *pQInfo) { qDebug("QInfo:%p start to free QInfo", pQInfo); - releaseQueryBuf(pQInfo->tableqinfoGroupInfo.numOfTables); + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + releaseQueryBuf(pRuntimeEnv->tableqinfoGroupInfo.numOfTables); teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); SQuery *pQuery = pQInfo->runtimeEnv.pQuery; if (pQuery != NULL) { - if (pQuery->sdata != NULL) { - for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { - tfree(pQuery->sdata[col]); - } - tfree(pQuery->sdata); - } - if (pQuery->fillVal != NULL) { tfree(pQuery->fillVal); } @@ -6954,20 +6426,16 @@ void freeQInfo(SQInfo *pQInfo) { taosArrayDestroy(pQuery->pGroupbyExpr->columnInfo); tfree(pQuery->pGroupbyExpr); } - - tfree(pQuery); } - doDestroyTableQueryInfo(&pQInfo->tableqinfoGroupInfo); + doDestroyTableQueryInfo(&pRuntimeEnv->tableqinfoGroupInfo); tfree(pQInfo->pBuf); tfree(pQInfo->sql); - tsdbDestroyTableGroup(&pQInfo->tableGroupInfo); - taosHashCleanup(pQInfo->arrTableIdInfo); - - taosArrayDestroy(pQInfo->groupResInfo.pRows); + tsdbDestroyTableGroup(&pQuery->tableGroupInfo); + taosArrayDestroy(pRuntimeEnv->groupResInfo.pRows); pQInfo->signature = 0; qDebug("QInfo:%p QInfo is freed", pQInfo); @@ -6975,41 +6443,20 @@ void freeQInfo(SQInfo *pQInfo) { tfree(pQInfo); } -size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) { - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - - /* - * get the file size and set the numOfRows to be the file size, since for tsComp query, - * the returned row size is equalled to 1 - * TODO handle the case that the file is too large to send back one time - */ - if (isTsCompQuery(pQuery) && (*numOfRows) > 0) { - struct stat fStat; - FILE *f = *(FILE **)pQuery->sdata[0]->data; - if ((f != NULL) && (fstat(fileno(f), &fStat) == 0)) { - *numOfRows = fStat.st_size; - return fStat.st_size; - } else { - qError("QInfo:%p failed to get file info, file:%p, reason:%s", pQInfo, f, strerror(errno)); - return 0; - } - } else { - return (size_t)(pQuery->resultRowSize * (*numOfRows)); - } -} - int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { // the remained number of retrieved rows, not the interpolated result + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; // load data from file to msg buffer if (isTsCompQuery(pQuery)) { - - FILE *f = *(FILE **)pQuery->sdata[0]->data; // TODO refactor + SColumnInfoData* pColInfoData = taosArrayGet(pRuntimeEnv->outputBuf->pDataBlock, 0); + FILE *f = *(FILE **)pColInfoData->pData; // TODO refactor // make sure file exist if (f) { off_t s = lseek(fileno(f), 0, SEEK_END); + assert(s == pRuntimeEnv->outputBuf->info.rows); qDebug("QInfo:%p ts comp data return, file:%p, size:%"PRId64, pQInfo, f, (uint64_t)s); if (fseek(f, 0, SEEK_SET) >= 0) { @@ -7024,30 +6471,30 @@ int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { assert(0); } + // dump error info if (s <= (sizeof(STSBufFileHeader) + sizeof(STSGroupBlockInfo) + 6 * sizeof(int32_t))) { qDump(data, s); - assert(0); } fclose(f); - *(FILE **)pQuery->sdata[0]->data = NULL; } // all data returned, set query over - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - setQueryStatus(pQuery, QUERY_OVER); + if (Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED)) { + setQueryStatus(pRuntimeEnv, QUERY_OVER); } } else { - doCopyQueryResultToMsg(pQInfo, (int32_t)pQuery->rec.rows, data); + doCopyQueryResultToMsg(pQInfo, (int32_t)pRuntimeEnv->outputBuf->info.rows, data); } - pQuery->rec.total += pQuery->rec.rows; - qDebug("QInfo:%p current numOfRes rows:%" PRId64 ", total:%" PRId64, pQInfo, pQuery->rec.rows, pQuery->rec.total); + pRuntimeEnv->resultInfo.total += pRuntimeEnv->outputBuf->info.rows; + qDebug("QInfo:%p current numOfRes rows:%d, total:%" PRId64, pQInfo, + pRuntimeEnv->outputBuf->info.rows, pRuntimeEnv->resultInfo.total); - if (pQuery->limit.limit > 0 && pQuery->limit.limit == pQuery->rec.total) { + if (pQuery->limit.limit > 0 && pQuery->limit.limit == pRuntimeEnv->resultInfo.total) { qDebug("QInfo:%p results limitation reached, limitation:%"PRId64, pQInfo, pQuery->limit.limit); - setQueryStatus(pQuery, QUERY_OVER); + setQueryStatus(pRuntimeEnv, QUERY_OVER); } return TSDB_CODE_SUCCESS; @@ -7086,139 +6533,6 @@ static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type } } -void buildTagQueryResult(SQInfo* pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; - - size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pQInfo); - assert(numOfGroup == 0 || numOfGroup == 1); - - if (numOfGroup == 0) { - return; - } - - SArray* pa = GET_TABLEGROUP(pQInfo, 0); - - size_t num = taosArrayGetSize(pa); - assert(num == pQInfo->tableqinfoGroupInfo.numOfTables); - - int32_t count = 0; - int32_t functionId = pQuery->pExpr1[0].base.functionId; - if (functionId == TSDB_FUNC_TID_TAG) { // return the tags & table Id - assert(pQuery->numOfOutput == 1); - - SExprInfo* pExprInfo = &pQuery->pExpr1[0]; - int32_t rsize = pExprInfo->bytes; - count = 0; - - int16_t bytes = pExprInfo->bytes; - int16_t type = pExprInfo->type; - - for(int32_t i = 0; i < pQuery->numOfTags; ++i) { - if (pQuery->tagColList[i].colId == pExprInfo->base.colInfo.colId) { - bytes = pQuery->tagColList[i].bytes; - type = pQuery->tagColList[i].type; - break; - } - } - - while(pQInfo->tableIndex < num && count < pQuery->rec.capacity) { - int32_t i = pQInfo->tableIndex++; - STableQueryInfo *item = taosArrayGetP(pa, i); - - char *output = pQuery->sdata[0]->data + count * rsize; - varDataSetLen(output, rsize - VARSTR_HEADER_SIZE); - - output = varDataVal(output); - STableId* id = TSDB_TABLEID(item->pTable); - - *(int16_t *)output = 0; - output += sizeof(int16_t); - - *(int64_t *)output = id->uid; // memory align problem, todo serialize - output += sizeof(id->uid); - - *(int32_t *)output = id->tid; - output += sizeof(id->tid); - - *(int32_t *)output = pQInfo->vgId; - output += sizeof(pQInfo->vgId); - - if (pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { - char* data = tsdbGetTableName(item->pTable); - memcpy(output, data, varDataTLen(data)); - } else { - char* data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.colInfo.colId, type, bytes); - doSetTagValueToResultBuf(output, data, type, bytes); - } - - count += 1; - } - - qDebug("QInfo:%p create (tableId, tag) info completed, rows:%d", pQInfo, count); - - } else if (functionId == TSDB_FUNC_COUNT) {// handle the "count(tbname)" query - *(int64_t*) pQuery->sdata[0]->data = num; - - count = 1; - SET_STABLE_QUERY_OVER(pQInfo); - qDebug("QInfo:%p create count(tbname) query, res:%d rows:1", pQInfo, count); - } else { // return only the tags|table name etc. - count = 0; - SSchema* tbnameSchema = tGetTbnameColumnSchema(); - - int32_t maxNumOfTables = (int32_t)pQuery->rec.capacity; - if (pQuery->limit.limit >= 0 && pQuery->limit.limit < pQuery->rec.capacity) { - maxNumOfTables = (int32_t)pQuery->limit.limit; - } - - while(pQInfo->tableIndex < num && count < maxNumOfTables) { - int32_t i = pQInfo->tableIndex++; - - // discard current result due to offset - if (pQuery->limit.offset > 0) { - pQuery->limit.offset -= 1; - continue; - } - - SExprInfo* pExprInfo = pQuery->pExpr1; - STableQueryInfo* item = taosArrayGetP(pa, i); - - char *data = NULL, *dst = NULL; - int16_t type = 0, bytes = 0; - for(int32_t j = 0; j < pQuery->numOfOutput; ++j) { - // not assign value in case of user defined constant output column - if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.colInfo.flag)) { - continue; - } - - if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { - bytes = tbnameSchema->bytes; - type = tbnameSchema->type; - - data = tsdbGetTableName(item->pTable); - dst = pQuery->sdata[j]->data + count * tbnameSchema->bytes; - } else { - type = pExprInfo[j].type; - bytes = pExprInfo[j].bytes; - - data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes); - dst = pQuery->sdata[j]->data + count * pExprInfo[j].bytes; - - } - - doSetTagValueToResultBuf(dst, data, type, bytes); - } - count += 1; - } - - qDebug("QInfo:%p create tag values results completed, rows:%d", pQInfo, count); - } - - pQuery->rec.rows = count; - setQueryStatus(pQuery, QUERY_COMPLETED); -} - static int64_t getQuerySupportBufSize(size_t numOfTables) { size_t s1 = sizeof(STableQueryInfo); size_t s2 = sizeof(SHashNode); diff --git a/src/query/src/qFill.c b/src/query/src/qFill.c index bc6376b80743bcad3e1b07f903c970eeb9e5dee2..2de1029396fb779798ea7c20da939b2153ca3dd3 100644 --- a/src/query/src/qFill.c +++ b/src/query/src/qFill.c @@ -23,18 +23,19 @@ #include "qFill.h" #include "qExtbuffer.h" #include "queryLog.h" +#include "qExecutor.h" #define FILL_IS_ASC_FILL(_f) ((_f)->order == TSDB_ORDER_ASC) #define DO_INTERPOLATION(_v1, _v2, _k1, _k2, _k) ((_v1) + ((_v2) - (_v1)) * (((double)(_k)) - ((double)(_k1))) / (((double)(_k2)) - ((double)(_k1)))) -static void setTagsValue(SFillInfo* pFillInfo, tFilePage** data, int32_t genRows) { +static void setTagsValue(SFillInfo* pFillInfo, void** data, int32_t genRows) { for(int32_t j = 0; j < pFillInfo->numOfCols; ++j) { SFillColInfo* pCol = &pFillInfo->pFillCol[j]; if (TSDB_COL_IS_NORMAL_COL(pCol->flag)) { continue; } - char* val1 = elePtrAt(data[j]->data, pCol->col.bytes, genRows); + char* val1 = elePtrAt(data[j], pCol->col.bytes, genRows); assert(pCol->tagIndex >= 0 && pCol->tagIndex < pFillInfo->numOfTags); SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex]; @@ -44,17 +45,17 @@ static void setTagsValue(SFillInfo* pFillInfo, tFilePage** data, int32_t genRows } } -static void setNullValueForRow(SFillInfo* pFillInfo, tFilePage** data, int32_t numOfCol, int32_t rowIndex) { +static void setNullValueForRow(SFillInfo* pFillInfo, void** data, int32_t numOfCol, int32_t rowIndex) { // the first are always the timestamp column, so start from the second column. for (int32_t i = 1; i < numOfCol; ++i) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - char* output = elePtrAt(data[i]->data, pCol->col.bytes, rowIndex); + char* output = elePtrAt(data[i], pCol->col.bytes, rowIndex); setNull(output, pCol->col.type, pCol->col.bytes); } } -static void doFillOneRowResult(SFillInfo* pFillInfo, tFilePage** data, char** srcData, int64_t ts, bool outOfBound) { +static void doFillOneRowResult(SFillInfo* pFillInfo, void** data, char** srcData, int64_t ts, bool outOfBound) { char* prev = pFillInfo->prevValues; char* next = pFillInfo->nextValues; @@ -63,7 +64,7 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, tFilePage** data, char** sr // set the primary timestamp column value int32_t index = pFillInfo->numOfCurrent; - char* val = elePtrAt(data[0]->data, TSDB_KEYSIZE, index); + char* val = elePtrAt(data[0], TSDB_KEYSIZE, index); *(TSKEY*) val = pFillInfo->currentKey; // set the other values @@ -77,7 +78,7 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, tFilePage** data, char** sr continue; } - char* output = elePtrAt(data[i]->data, pCol->col.bytes, index); + char* output = elePtrAt(data[i], pCol->col.bytes, index); assignVal(output, p + pCol->col.offset, pCol->col.bytes, pCol->col.type); } } else { // no prev value yet, set the value for NULL @@ -93,7 +94,7 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, tFilePage** data, char** sr continue; } - char* output = elePtrAt(data[i]->data, pCol->col.bytes, index); + char* output = elePtrAt(data[i], pCol->col.bytes, index); assignVal(output, p + pCol->col.offset, pCol->col.bytes, pCol->col.type); } } else { // no prev value yet, set the value for NULL @@ -111,7 +112,7 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, tFilePage** data, char** sr int16_t type = pCol->col.type; int16_t bytes = pCol->col.bytes; - char *val1 = elePtrAt(data[i]->data, pCol->col.bytes, index); + char *val1 = elePtrAt(data[i], pCol->col.bytes, index); if (type == TSDB_DATA_TYPE_BINARY|| type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BOOL) { setNull(val1, pCol->col.type, bytes); continue; @@ -132,7 +133,7 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, tFilePage** data, char** sr continue; } - char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, index); + char* val1 = elePtrAt(data[i], pCol->col.bytes, index); assignVal(val1, (char*)&pCol->fillVal.i, pCol->col.bytes, pCol->col.type); } } @@ -162,7 +163,7 @@ static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, char** srcData, char* bu } } -static int32_t fillResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t outputRows) { +static int32_t fillResultImpl(SFillInfo* pFillInfo, void** data, int32_t outputRows) { pFillInfo->numOfCurrent = 0; char** srcData = pFillInfo->pData; @@ -213,7 +214,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t ou continue; } - char* output = elePtrAt(data[i]->data, pCol->col.bytes, pFillInfo->numOfCurrent); + char* output = elePtrAt(data[i], pCol->col.bytes, pFillInfo->numOfCurrent); char* src = elePtrAt(srcData[i], pCol->col.bytes, pFillInfo->index); if (i == 0 || (pCol->functionId != TSDB_FUNC_COUNT && !isNull(src, pCol->col.type)) || @@ -255,7 +256,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t ou return pFillInfo->numOfCurrent; } -static int64_t appendFilledResult(SFillInfo* pFillInfo, tFilePage** output, int64_t resultCapacity) { +static int64_t appendFilledResult(SFillInfo* pFillInfo, void** output, int64_t resultCapacity) { /* * These data are generated according to fill strategy, since the current timestamp is out of the time window of * real result set. Note that we need to keep the direct previous result rows, to generated the filled data. @@ -278,7 +279,7 @@ static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t int32_t k = 0; for (int32_t i = 0; i < numOfCols; ++i) { SFillColInfo* pColInfo = &pFillInfo->pFillCol[i]; - pFillInfo->pData[i] = calloc(1, pColInfo->col.bytes * capacity); + pFillInfo->pData[i] = NULL; if (TSDB_COL_IS_TAG(pColInfo->flag)) { bool exists = false; @@ -356,6 +357,10 @@ SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int3 pFillInfo->rowSize = setTagColumnInfo(pFillInfo, pFillInfo->numOfCols, pFillInfo->alloc); assert(pFillInfo->rowSize > 0); + for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { + pFillInfo->pData[i] = malloc(pFillInfo->pFillCol[i].col.bytes * pFillInfo->alloc); + } + return pFillInfo; } @@ -375,11 +380,16 @@ void* taosDestroyFillInfo(SFillInfo* pFillInfo) { tfree(pFillInfo->prevValues); tfree(pFillInfo->nextValues); - tfree(pFillInfo->pTags); - + + for(int32_t i = 0; i < pFillInfo->numOfTags; ++i) { + tfree(pFillInfo->pTags[i].tagVal); + } + for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { tfree(pFillInfo->pData[i]); } + + tfree(pFillInfo->pTags); tfree(pFillInfo->pData); tfree(pFillInfo->pFillCol); @@ -413,10 +423,19 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) } } -// copy the data into source data buffer -void taosFillSetDataBlockFromFilePage(SFillInfo* pFillInfo, const tFilePage** pInput) { +void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const SSDataBlock* pInput) { for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - memcpy(pFillInfo->pData[i], pInput[i]->data, pFillInfo->numOfRows * pFillInfo->pFillCol[i].col.bytes); + SColumnInfoData* pColData = taosArrayGet(pInput->pDataBlock, i); +// pFillInfo->pData[i] = pColData->pData; + if (pInput->info.rows > pFillInfo->alloc) { + char* t = realloc(pFillInfo->pData[i], pColData->info.bytes * pInput->info.rows); + assert(t != NULL); + + pFillInfo->pData[i] = t; + pFillInfo->alloc = pInput->info.rows; + } + + memcpy(pFillInfo->pData[i], pColData->pData, pColData->info.bytes * pInput->info.rows); } } @@ -427,12 +446,20 @@ void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, const tFilePage* SFillColInfo* pCol = &pFillInfo->pFillCol[i]; const char* data = pInput->data + pCol->col.offset * pInput->num; - memcpy(pFillInfo->pData[i], data, (size_t)(pInput->num * pCol->col.bytes)); + if (pInput->num > pFillInfo->alloc) { + char* t = realloc(pFillInfo->pData[i], (size_t)(pCol->col.bytes * pInput->num)); + assert(t != NULL); + + pFillInfo->pData[i] = t; + pFillInfo->alloc = (int32_t)pInput->num; + } + + memcpy(pFillInfo->pData[i], data, (size_t)(pCol->col.bytes * pInput->num)); if (TSDB_COL_IS_TAG(pCol->flag)) { // copy the tag value to tag value buffer SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex]; assert (pTag->col.colId == pCol->col.colId); - memcpy(pTag->tagVal, data, pCol->col.bytes); + memcpy(pTag->tagVal, data, pCol->col.bytes); // TODO not memcpy?? } } } @@ -490,7 +517,7 @@ int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* return TSDB_CODE_SUCCESS; } -int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity) { +int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t capacity) { int32_t remain = taosNumOfRemainRows(pFillInfo); int64_t numOfRes = getNumOfResultsAfterFillGap(pFillInfo, pFillInfo->end, capacity); diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index bd6af3a2ebd91718984ecf93e20a80f3269ac5ff..891084f642c4d18f27d3eebf5ac56b475dc460b5 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -135,28 +135,33 @@ tSQLExpr *tSqlExprIdValueCreate(SStrToken *pToken, int32_t optrType) { if (optrType == TK_INTEGER || optrType == TK_STRING || optrType == TK_FLOAT || optrType == TK_BOOL) { toTSDBType(pToken->type); - tVariantCreate(&pSqlExpr->val, pToken); - pSqlExpr->nSQLOptr = optrType; + tVariantCreate(&pSqlExpr->value, pToken); + pSqlExpr->tokenId = optrType; + pSqlExpr->type = SQL_NODE_VALUE; } else if (optrType == TK_NOW) { // use microsecond by default - pSqlExpr->val.i64 = taosGetTimestamp(TSDB_TIME_PRECISION_MICRO); - pSqlExpr->val.nType = TSDB_DATA_TYPE_BIGINT; - pSqlExpr->nSQLOptr = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond + pSqlExpr->value.i64 = taosGetTimestamp(TSDB_TIME_PRECISION_MICRO); + pSqlExpr->value.nType = TSDB_DATA_TYPE_BIGINT; + pSqlExpr->tokenId = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond + pSqlExpr->type = SQL_NODE_VALUE; } else if (optrType == TK_VARIABLE) { - int32_t ret = parseAbsoluteDuration(pToken->z, pToken->n, &pSqlExpr->val.i64); + int32_t ret = parseAbsoluteDuration(pToken->z, pToken->n, &pSqlExpr->value.i64); if (ret != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; } - pSqlExpr->val.nType = TSDB_DATA_TYPE_BIGINT; - pSqlExpr->nSQLOptr = TK_TIMESTAMP; - } else { // it must be the column name (tk_id) if it is not the number + pSqlExpr->value.nType = TSDB_DATA_TYPE_BIGINT; + pSqlExpr->tokenId = TK_TIMESTAMP; + pSqlExpr->type = SQL_NODE_VALUE; + } else { + // Here it must be the column name (tk_id) if it is not a number or string. assert(optrType == TK_ID || optrType == TK_ALL); if (pToken != NULL) { pSqlExpr->colInfo = *pToken; } - pSqlExpr->nSQLOptr = optrType; + pSqlExpr->tokenId = optrType; + pSqlExpr->type = SQL_NODE_TABLE_COLUMN; } return pSqlExpr; @@ -167,19 +172,22 @@ tSQLExpr *tSqlExprIdValueCreate(SStrToken *pToken, int32_t optrType) { * function name is denoted by pFunctionToken */ tSQLExpr *tSqlExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType) { - if (pFuncToken == NULL) return NULL; + if (pFuncToken == NULL) { + return NULL; + } tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr)); - pExpr->nSQLOptr = optType; - pExpr->pParam = pList; + pExpr->tokenId = optType; + pExpr->type = SQL_NODE_SQLFUNCTION; + pExpr->pParam = pList; int32_t len = (int32_t)((endToken->z + endToken->n) - pFuncToken->z); - pExpr->operand.z = pFuncToken->z; + pExpr->operand = (*pFuncToken); - pExpr->operand.n = len; // raw field name - pExpr->operand.type = pFuncToken->type; + pExpr->token.n = len; + pExpr->token.z = pFuncToken->z; + pExpr->token.type = pFuncToken->type; - pExpr->token = pExpr->operand; return pExpr; } @@ -190,6 +198,7 @@ tSQLExpr *tSqlExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SSt tSQLExpr *tSqlExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr)); + pExpr->type = SQL_NODE_EXPR; if (pLeft != NULL && pRight != NULL && (optrType != TK_IN)) { char* endPos = pRight->token.z + pRight->token.n; pExpr->token.z = pLeft->token.z; @@ -203,32 +212,33 @@ tSQLExpr *tSqlExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { * if a token is noted as the TK_TIMESTAMP, the time precision is microsecond * Otherwise, the time precision is adaptive, determined by the time precision from databases. */ - if ((pLeft->nSQLOptr == TK_INTEGER && pRight->nSQLOptr == TK_INTEGER) || - (pLeft->nSQLOptr == TK_TIMESTAMP && pRight->nSQLOptr == TK_TIMESTAMP)) { - pExpr->val.nType = TSDB_DATA_TYPE_BIGINT; - pExpr->nSQLOptr = pLeft->nSQLOptr; + if ((pLeft->tokenId == TK_INTEGER && pRight->tokenId == TK_INTEGER) || + (pLeft->tokenId == TK_TIMESTAMP && pRight->tokenId == TK_TIMESTAMP)) { + pExpr->value.nType = TSDB_DATA_TYPE_BIGINT; + pExpr->tokenId = pLeft->tokenId; + pExpr->type = SQL_NODE_VALUE; switch (optrType) { case TK_PLUS: { - pExpr->val.i64 = pLeft->val.i64 + pRight->val.i64; + pExpr->value.i64 = pLeft->value.i64 + pRight->value.i64; break; } case TK_MINUS: { - pExpr->val.i64 = pLeft->val.i64 - pRight->val.i64; + pExpr->value.i64 = pLeft->value.i64 - pRight->value.i64; break; } case TK_STAR: { - pExpr->val.i64 = pLeft->val.i64 * pRight->val.i64; + pExpr->value.i64 = pLeft->value.i64 * pRight->value.i64; break; } case TK_DIVIDE: { - pExpr->nSQLOptr = TK_FLOAT; - pExpr->val.nType = TSDB_DATA_TYPE_DOUBLE; - pExpr->val.dKey = (double)pLeft->val.i64 / pRight->val.i64; + pExpr->tokenId = TK_FLOAT; + pExpr->value.nType = TSDB_DATA_TYPE_DOUBLE; + pExpr->value.dKey = (double)pLeft->value.i64 / pRight->value.i64; break; } case TK_REM: { - pExpr->val.i64 = pLeft->val.i64 % pRight->val.i64; + pExpr->value.i64 = pLeft->value.i64 % pRight->value.i64; break; } } @@ -236,33 +246,35 @@ tSQLExpr *tSqlExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { tSqlExprDestroy(pLeft); tSqlExprDestroy(pRight); - } else if ((pLeft->nSQLOptr == TK_FLOAT && pRight->nSQLOptr == TK_INTEGER) || (pLeft->nSQLOptr == TK_INTEGER && pRight->nSQLOptr == TK_FLOAT) || - (pLeft->nSQLOptr == TK_FLOAT && pRight->nSQLOptr == TK_FLOAT)) { - pExpr->val.nType = TSDB_DATA_TYPE_DOUBLE; - pExpr->nSQLOptr = TK_FLOAT; + } else if ((pLeft->tokenId == TK_FLOAT && pRight->tokenId == TK_INTEGER) || + (pLeft->tokenId == TK_INTEGER && pRight->tokenId == TK_FLOAT) || + (pLeft->tokenId == TK_FLOAT && pRight->tokenId == TK_FLOAT)) { + pExpr->value.nType = TSDB_DATA_TYPE_DOUBLE; + pExpr->tokenId = TK_FLOAT; + pExpr->type = SQL_NODE_VALUE; - double left = (pLeft->val.nType == TSDB_DATA_TYPE_DOUBLE) ? pLeft->val.dKey : pLeft->val.i64; - double right = (pRight->val.nType == TSDB_DATA_TYPE_DOUBLE) ? pRight->val.dKey : pRight->val.i64; + double left = (pLeft->value.nType == TSDB_DATA_TYPE_DOUBLE) ? pLeft->value.dKey : pLeft->value.i64; + double right = (pRight->value.nType == TSDB_DATA_TYPE_DOUBLE) ? pRight->value.dKey : pRight->value.i64; switch (optrType) { case TK_PLUS: { - pExpr->val.dKey = left + right; + pExpr->value.dKey = left + right; break; } case TK_MINUS: { - pExpr->val.dKey = left - right; + pExpr->value.dKey = left - right; break; } case TK_STAR: { - pExpr->val.dKey = left * right; + pExpr->value.dKey = left * right; break; } case TK_DIVIDE: { - pExpr->val.dKey = left / right; + pExpr->value.dKey = left / right; break; } case TK_REM: { - pExpr->val.dKey = left - ((int64_t)(left / right)) * right; + pExpr->value.dKey = left - ((int64_t)(left / right)) * right; break; } } @@ -271,21 +283,21 @@ tSQLExpr *tSqlExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { tSqlExprDestroy(pRight); } else { - pExpr->nSQLOptr = optrType; + pExpr->tokenId = optrType; pExpr->pLeft = pLeft; pExpr->pRight = pRight; } } else if (optrType == TK_IN) { - pExpr->nSQLOptr = optrType; + pExpr->tokenId = optrType; pExpr->pLeft = pLeft; tSQLExpr *pRSub = calloc(1, sizeof(tSQLExpr)); - pRSub->nSQLOptr = TK_SET; // TODO refactor ..... + pRSub->tokenId = TK_SET; // TODO refactor ..... pRSub->pParam = (tSQLExprList *)pRight; pExpr->pRight = pRSub; } else { - pExpr->nSQLOptr = optrType; + pExpr->tokenId = optrType; pExpr->pLeft = pLeft; if (pLeft != NULL && pRight == NULL) { @@ -325,8 +337,8 @@ void tSqlExprNodeDestroy(tSQLExpr *pExpr) { return; } - if (pExpr->nSQLOptr == TK_STRING) { - tVariantDestroy(&pExpr->val); + if (pExpr->tokenId == TK_STRING) { + tVariantDestroy(&pExpr->value); } tSqlExprListDestroy(pExpr->pParam); @@ -538,8 +550,8 @@ void tSqlSetColumnType(TAOS_FIELD *pField, SStrToken *type) { /* * extract the select info out of sql string */ -SQuerySQL *tSetQuerySqlElems(SStrToken *pSelectToken, tSQLExprList *pSelection, SArray *pFrom, tSQLExpr *pWhere, - SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, +SQuerySQL *tSetQuerySqlNode(SStrToken *pSelectToken, tSQLExprList *pSelection, SArray *pFrom, tSQLExpr *pWhere, + SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, SSessionWindowVal *pSession, SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, SLimitVal *pGLimit) { assert(pSelection != NULL); @@ -562,14 +574,17 @@ SQuerySQL *tSetQuerySqlElems(SStrToken *pSelectToken, tSQLExprList *pSelection, } if (pInterval != NULL) { - pQuery->interval = pInterval->interval; - pQuery->offset = pInterval->offset; + pQuery->interval = *pInterval; } if (pSliding != NULL) { pQuery->sliding = *pSliding; } + if (pSession != NULL) { + pQuery->sessionVal = *pSession; + } + pQuery->fillType = pFill; return pQuery; } diff --git a/src/query/src/qPercentile.c b/src/query/src/qPercentile.c index 523fa42547e34e9f0bb9ac23868ab3dc67eeec34..d36664617244cb0059b4e291da88260fac9db225 100644 --- a/src/query/src/qPercentile.c +++ b/src/query/src/qPercentile.c @@ -254,7 +254,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, resetSlotInfo(pBucket); - int32_t ret = createDiskbasedResultBuffer(&pBucket->pBuffer, pBucket->bytes, pBucket->bufPageSize, pBucket->bufPageSize * 512, NULL); + int32_t ret = createDiskbasedResultBuffer(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, NULL); if (ret != TSDB_CODE_SUCCESS) { tMemBucketDestroy(pBucket); return NULL; diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index d45e76c2fd07f37afa0ffad0f9db65d5e1948c43..ed7f8c67194269b20e21809ff0948662f4f25147 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -9,8 +9,7 @@ #define GET_DATA_PAYLOAD(_p) ((char *)(_p)->pData + POINTER_BYTES) #define NO_IN_MEM_AVAILABLE_PAGES(_b) (listNEles((_b)->lruList) >= (_b)->inMemPages) -int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t rowSize, int32_t pagesize, - int32_t inMemBufSize, const void* handle) { +int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t pagesize, int32_t inMemBufSize, const void* handle) { *pResultBuf = calloc(1, sizeof(SDiskbasedResultBuf)); SDiskbasedResultBuf* pResBuf = *pResultBuf; @@ -31,7 +30,6 @@ int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t ro // at least more than 2 pages must be in memory assert(inMemBufSize >= pagesize * 2); - pResBuf->numOfRowsPerPage = (pagesize - sizeof(tFilePage)) / rowSize; pResBuf->lruList = tdListNew(POINTER_BYTES); // init id hash table @@ -387,8 +385,6 @@ void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi) { pResultBuf->statis.releasePages += 1; } -size_t getNumOfRowsPerPage(const SDiskbasedResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; } - size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->groupSet); } size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf) { return (size_t)pResultBuf->totalBufSize; } diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c index 62f5606609e9a4e0061f23a9b32efd3ca183490b..4adb91326586c5fba713928511934ece9a04dbb3 100644 --- a/src/query/src/qTokenizer.c +++ b/src/query/src/qTokenizer.c @@ -139,6 +139,7 @@ static SKeyword keywordTable[] = { {"FROM", TK_FROM}, {"VARIABLE", TK_VARIABLE}, {"INTERVAL", TK_INTERVAL}, + {"SESSION", TK_SESSION}, {"FILL", TK_FILL}, {"SLIDING", TK_SLIDING}, {"ORDER", TK_ORDER}, @@ -200,25 +201,6 @@ static SKeyword keywordTable[] = { {"TRIGGER", TK_TRIGGER}, {"VIEW", TK_VIEW}, {"ALL", TK_ALL}, - {"COUNT", TK_COUNT}, - {"SUM", TK_SUM}, - {"AVG", TK_AVG}, - {"MIN", TK_MIN}, - {"MAX", TK_MAX}, - {"FIRST", TK_FIRST}, - {"LAST", TK_LAST}, - {"TOP", TK_TOP}, - {"BOTTOM", TK_BOTTOM}, - {"STDDEV", TK_STDDEV}, - {"PERCENTILE", TK_PERCENTILE}, - {"APERCENTILE", TK_APERCENTILE}, - {"LEASTSQUARES", TK_LEASTSQUARES}, - {"HISTOGRAM", TK_HISTOGRAM}, - {"DIFF", TK_DIFF}, - {"SPREAD", TK_SPREAD}, - {"TWA", TK_TWA}, - {"INTERP", TK_INTERP}, - {"LAST_ROW", TK_LAST_ROW}, {"SEMI", TK_SEMI}, {"NONE", TK_NONE}, {"PREV", TK_PREV}, @@ -228,17 +210,10 @@ static SKeyword keywordTable[] = { {"TBNAME", TK_TBNAME}, {"JOIN", TK_JOIN}, {"METRICS", TK_METRICS}, - {"TBID", TK_TBID}, {"STABLE", TK_STABLE}, {"FILE", TK_FILE}, {"VNODES", TK_VNODES}, {"UNION", TK_UNION}, - {"RATE", TK_RATE}, - {"IRATE", TK_IRATE}, - {"SUM_RATE", TK_SUM_RATE}, - {"SUM_IRATE", TK_SUM_IRATE}, - {"AVG_RATE", TK_AVG_RATE}, - {"AVG_IRATE", TK_AVG_IRATE}, {"CACHELAST", TK_CACHELAST}, {"DISTINCT", TK_DISTINCT}, {"PARTITIONS", TK_PARTITIONS}, @@ -300,7 +275,7 @@ int tSQLKeywordCode(const char* z, int n) { * Return the length of the token that begins at z[0]. * Store the token type in *type before returning. */ -uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { +uint32_t tSQLGetToken(char* z, uint32_t* tokenId) { uint32_t i; switch (*z) { case ' ': @@ -310,121 +285,121 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { case '\r': { for (i = 1; isspace(z[i]); i++) { } - *tokenType = TK_SPACE; + *tokenId = TK_SPACE; return i; } case ':': { - *tokenType = TK_COLON; + *tokenId = TK_COLON; return 1; } case '-': { if (z[1] == '-') { for (i = 2; z[i] && z[i] != '\n'; i++) { } - *tokenType = TK_COMMENT; + *tokenId = TK_COMMENT; return i; } - *tokenType = TK_MINUS; + *tokenId = TK_MINUS; return 1; } case '(': { - *tokenType = TK_LP; + *tokenId = TK_LP; return 1; } case ')': { - *tokenType = TK_RP; + *tokenId = TK_RP; return 1; } case ';': { - *tokenType = TK_SEMI; + *tokenId = TK_SEMI; return 1; } case '+': { - *tokenType = TK_PLUS; + *tokenId = TK_PLUS; return 1; } case '*': { - *tokenType = TK_STAR; + *tokenId = TK_STAR; return 1; } case '/': { if (z[1] != '*' || z[2] == 0) { - *tokenType = TK_SLASH; + *tokenId = TK_SLASH; return 1; } for (i = 3; z[i] && (z[i] != '/' || z[i - 1] != '*'); i++) { } if (z[i]) i++; - *tokenType = TK_COMMENT; + *tokenId = TK_COMMENT; return i; } case '%': { - *tokenType = TK_REM; + *tokenId = TK_REM; return 1; } case '=': { - *tokenType = TK_EQ; + *tokenId = TK_EQ; return 1 + (z[1] == '='); } case '<': { if (z[1] == '=') { - *tokenType = TK_LE; + *tokenId = TK_LE; return 2; } else if (z[1] == '>') { - *tokenType = TK_NE; + *tokenId = TK_NE; return 2; } else if (z[1] == '<') { - *tokenType = TK_LSHIFT; + *tokenId = TK_LSHIFT; return 2; } else { - *tokenType = TK_LT; + *tokenId = TK_LT; return 1; } } case '>': { if (z[1] == '=') { - *tokenType = TK_GE; + *tokenId = TK_GE; return 2; } else if (z[1] == '>') { - *tokenType = TK_RSHIFT; + *tokenId = TK_RSHIFT; return 2; } else { - *tokenType = TK_GT; + *tokenId = TK_GT; return 1; } } case '!': { if (z[1] != '=') { - *tokenType = TK_ILLEGAL; + *tokenId = TK_ILLEGAL; return 2; } else { - *tokenType = TK_NE; + *tokenId = TK_NE; return 2; } } case '|': { if (z[1] != '|') { - *tokenType = TK_BITOR; + *tokenId = TK_BITOR; return 1; } else { - *tokenType = TK_CONCAT; + *tokenId = TK_CONCAT; return 2; } } case ',': { - *tokenType = TK_COMMA; + *tokenId = TK_COMMA; return 1; } case '&': { - *tokenType = TK_BITAND; + *tokenId = TK_BITAND; return 1; } case '~': { - *tokenType = TK_BITNOT; + *tokenId = TK_BITNOT; return 1; } case '?': { - *tokenType = TK_QUESTION; + *tokenId = TK_QUESTION; return 1; } case '\'': @@ -450,7 +425,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { if (z[i]) i++; if (strEnd) { - *tokenType = TK_STRING; + *tokenId = TK_STRING; return i; } @@ -474,10 +449,10 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { } } - *tokenType = TK_FLOAT; + *tokenId = TK_FLOAT; return i; } else { - *tokenType = TK_DOT; + *tokenId = TK_DOT; return 1; } } @@ -486,7 +461,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { char next = z[1]; if (next == 'b') { // bin number - *tokenType = TK_BIN; + *tokenId = TK_BIN; for (i = 2; (z[i] == '0' || z[i] == '1'); ++i) { } @@ -496,7 +471,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { return i; } else if (next == 'x') { //hex number - *tokenType = TK_HEX; + *tokenId = TK_HEX; for (i = 2; isdigit(z[i]) || (z[i] >= 'a' && z[i] <= 'f') || (z[i] >= 'A' && z[i] <= 'F'); ++i) { } @@ -516,7 +491,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { case '7': case '8': case '9': { - *tokenType = TK_INTEGER; + *tokenId = TK_INTEGER; for (i = 1; isdigit(z[i]); i++) { } @@ -526,7 +501,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { z[i] == 'U' || z[i] == 'A' || z[i] == 'S' || z[i] == 'M' || z[i] == 'H' || z[i] == 'D' || z[i] == 'N' || z[i] == 'Y' || z[i] == 'W') && (isIdChar[(uint8_t)z[i + 1]] == 0)) { - *tokenType = TK_VARIABLE; + *tokenId = TK_VARIABLE; i += 1; return i; } @@ -537,12 +512,12 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { while (isdigit(z[i])) { i++; } - *tokenType = TK_FLOAT; + *tokenId = TK_FLOAT; seg++; } if (seg == 4) { // ip address - *tokenType = TK_IPTOKEN; + *tokenId = TK_IPTOKEN; return i; } @@ -552,14 +527,14 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { while (isdigit(z[i])) { i++; } - *tokenType = TK_FLOAT; + *tokenId = TK_FLOAT; } return i; } case '[': { for (i = 1; z[i] && z[i - 1] != ']'; i++) { } - *tokenType = TK_ID; + *tokenId = TK_ID; return i; } case 'T': @@ -570,7 +545,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { } if ((i == 4 && strncasecmp(z, "true", 4) == 0) || (i == 5 && strncasecmp(z, "false", 5) == 0)) { - *tokenType = TK_BOOL; + *tokenId = TK_BOOL; return i; } } @@ -580,12 +555,12 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { } for (i = 1; ((z[i] & 0x80) == 0) && isIdChar[(uint8_t) z[i]]; i++) { } - *tokenType = tSQLKeywordCode(z, i); + *tokenId = tSQLKeywordCode(z, i); return i; } } - *tokenType = TK_ILLEGAL; + *tokenId = TK_ILLEGAL; return 0; } diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index b7a7fd28e9dc2f013457e22446ff7bc41d3fef6c..1e7d3c9b58d3594280edc294e2147e7c908b9d0f 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -22,6 +22,7 @@ #include "tbuffer.h" #include "tlosertree.h" #include "queryLog.h" +#include "tscompression.h" typedef struct SCompSupporter { STableQueryInfo **pTableQueryInfo; @@ -41,12 +42,11 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery) { } int32_t initResultRowInfo(SResultRowInfo *pResultRowInfo, int32_t size, int16_t type) { - pResultRowInfo->capacity = size; - - pResultRowInfo->type = type; - pResultRowInfo->curIndex = -1; + pResultRowInfo->type = type; pResultRowInfo->size = 0; pResultRowInfo->prevSKey = TSKEY_INITIAL_VAL; + pResultRowInfo->curIndex = -1; + pResultRowInfo->capacity = size; pResultRowInfo->pResult = calloc(pResultRowInfo->capacity, POINTER_BYTES); if (pResultRowInfo->pResult == NULL) { @@ -135,20 +135,22 @@ void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16 if (pResultRow->pageId >= 0) { tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResultRow->pageId); + int16_t offset = 0; for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) { SResultRowCellInfo *pResultInfo = &pResultRow->pCellInfo[i]; - char * s = getPosInResultPage(pRuntimeEnv, i, pResultRow, page); - size_t size = pRuntimeEnv->pQuery->pExpr1[i].bytes; + int16_t size = pRuntimeEnv->pQuery->pExpr1[i].bytes; + char * s = getPosInResultPage(pRuntimeEnv->pQuery, page, pResultRow->offset, offset); memset(s, 0, size); + offset += size; RESET_RESULT_INFO(pResultInfo); } } pResultRow->numOfRows = 0; pResultRow->pageId = -1; - pResultRow->rowId = -1; + pResultRow->offset = -1; pResultRow->closed = false; if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { @@ -158,13 +160,15 @@ void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16 } } -SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index) { - assert(index >= 0 && index < pRuntimeEnv->pQuery->numOfOutput); - return (SResultRowCellInfo*)((char*) pRow->pCellInfo + pRuntimeEnv->rowCellInfoOffset[index]); +// TODO refactor: use macro +SResultRowCellInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset) { + assert(index >= 0 && offset != NULL); + return (SResultRowCellInfo*)((char*) pRow->pCellInfo + offset[index]); } size_t getResultRowSize(SQueryRuntimeEnv* pRuntimeEnv) { - return (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultRowCellInfo)) + pRuntimeEnv->interBufSize + sizeof(SResultRow); + SQuery* pQuery = pRuntimeEnv->pQuery; + return (pQuery->numOfOutput * sizeof(SResultRowCellInfo)) + pQuery->interBufSize + sizeof(SResultRow); } SResultRowPool* initResultRowPool(size_t size) { @@ -340,18 +344,18 @@ void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) { pGroupResInfo->index = 0; } -void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo, int32_t offset) { +void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo) { if (pGroupResInfo->pRows != NULL) { taosArrayDestroy(pGroupResInfo->pRows); } pGroupResInfo->pRows = taosArrayFromList(pResultInfo->pResult, pResultInfo->size, POINTER_BYTES); - pGroupResInfo->index = offset; + pGroupResInfo->index = 0; assert(pGroupResInfo->index <= getNumOfTotalRes(pGroupResInfo)); } -bool hasRemainData(SGroupResInfo* pGroupResInfo) { +bool hasRemainDataInCurrentGroup(SGroupResInfo* pGroupResInfo) { if (pGroupResInfo->pRows == NULL) { return false; } @@ -359,6 +363,14 @@ bool hasRemainData(SGroupResInfo* pGroupResInfo) { return pGroupResInfo->index < taosArrayGetSize(pGroupResInfo->pRows); } +bool hasRemainData(SGroupResInfo* pGroupResInfo) { + if (hasRemainDataInCurrentGroup(pGroupResInfo)) { + return true; + } + + return pGroupResInfo->currentGroup < pGroupResInfo->totalGroup; +} + bool incNextGroup(SGroupResInfo* pGroupResInfo) { return (++pGroupResInfo->currentGroup) < pGroupResInfo->totalGroup; } @@ -372,7 +384,7 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo) { return (int32_t) taosArrayGetSize(pGroupResInfo->pRows); } -static int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow *pResultRow) { +static int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow *pResultRow, int32_t* rowCellInfoOffset) { SQuery* pQuery = pRuntimeEnv->pQuery; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { @@ -386,7 +398,7 @@ static int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow continue; } - SResultRowCellInfo *pResultInfo = getResultCell(pRuntimeEnv, pResultRow, j); + SResultRowCellInfo *pResultInfo = getResultCell(pResultRow, j, rowCellInfoOffset); assert(pResultInfo != NULL); if (pResultInfo->numOfRes > 0) { @@ -437,7 +449,8 @@ static int32_t tableResultComparFn(const void *pLeft, const void *pRight, void * } } -static int32_t mergeIntoGroupResultImpl(SQueryRuntimeEnv *pRuntimeEnv, SGroupResInfo* pGroupResInfo, SArray *pTableList, void* qinfo) { +static int32_t mergeIntoGroupResultImpl(SQueryRuntimeEnv *pRuntimeEnv, SGroupResInfo* pGroupResInfo, SArray *pTableList, + int32_t* rowCellInfoOffset) { bool ascQuery = QUERY_IS_ASC_QUERY(pRuntimeEnv->pQuery); int32_t code = TSDB_CODE_SUCCESS; @@ -455,7 +468,7 @@ static int32_t mergeIntoGroupResultImpl(SQueryRuntimeEnv *pRuntimeEnv, SGroupRes pTableQueryInfoList = malloc(POINTER_BYTES * size); if (pTableQueryInfoList == NULL || posList == NULL || pGroupResInfo->pRows == NULL || pGroupResInfo->pRows == NULL) { - qError("QInfo:%p failed alloc memory", qinfo); + qError("QInfo:%p failed alloc memory", pRuntimeEnv->qinfo); code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _end; } @@ -491,7 +504,7 @@ static int32_t mergeIntoGroupResultImpl(SQueryRuntimeEnv *pRuntimeEnv, SGroupRes SResultRowInfo *pWindowResInfo = &pTableQueryInfoList[tableIndex]->resInfo; SResultRow *pWindowRes = getResultRow(pWindowResInfo, cs.rowIndex[tableIndex]); - int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pWindowRes); + int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pWindowRes, rowCellInfoOffset); if (num <= 0) { cs.rowIndex[tableIndex] += 1; @@ -527,7 +540,7 @@ static int32_t mergeIntoGroupResultImpl(SQueryRuntimeEnv *pRuntimeEnv, SGroupRes int64_t endt = taosGetTimestampMs(); - qDebug("QInfo:%p result merge completed for group:%d, elapsed time:%" PRId64 " ms", qinfo, + qDebug("QInfo:%p result merge completed for group:%d, elapsed time:%" PRId64 " ms", pRuntimeEnv->qinfo, pGroupResInfo->currentGroup, endt - startt); _end: @@ -538,13 +551,13 @@ static int32_t mergeIntoGroupResultImpl(SQueryRuntimeEnv *pRuntimeEnv, SGroupRes return code; } -int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, SQInfo *pQInfo) { +int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, SQueryRuntimeEnv* pRuntimeEnv, int32_t* offset) { int64_t st = taosGetTimestampUs(); while (pGroupResInfo->currentGroup < pGroupResInfo->totalGroup) { - SArray *group = GET_TABLEGROUP(pQInfo, pGroupResInfo->currentGroup); + SArray *group = GET_TABLEGROUP(pRuntimeEnv, pGroupResInfo->currentGroup); - int32_t ret = mergeIntoGroupResultImpl(&pQInfo->runtimeEnv, pGroupResInfo, group, pQInfo); + int32_t ret = mergeIntoGroupResultImpl(pRuntimeEnv, pGroupResInfo, group, offset); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -554,19 +567,83 @@ int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, SQInfo *pQInfo) { break; } - qDebug("QInfo:%p no result in group %d, continue", pQInfo, pGroupResInfo->currentGroup); + qDebug("QInfo:%p no result in group %d, continue", pRuntimeEnv->qinfo, pGroupResInfo->currentGroup); cleanupGroupResInfo(pGroupResInfo); incNextGroup(pGroupResInfo); } - if (pGroupResInfo->currentGroup >= pGroupResInfo->totalGroup && !hasRemainData(pGroupResInfo)) { - SET_STABLE_QUERY_OVER(pQInfo); - } - int64_t elapsedTime = taosGetTimestampUs() - st; - qDebug("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", pQInfo, + qDebug("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", pRuntimeEnv->qinfo, pGroupResInfo->currentGroup, pGroupResInfo->totalGroup, elapsedTime); - pQInfo->runtimeEnv.summary.firstStageMergeTime += elapsedTime; +// pQInfo->summary.firstStageMergeTime += elapsedTime; return TSDB_CODE_SUCCESS; } + +void blockDistInfoToBinary(STableBlockDist* pDist, struct SBufferWriter* bw) { + tbufWriteUint32(bw, pDist->numOfTables); + tbufWriteUint16(bw, pDist->numOfFiles); + tbufWriteUint64(bw, pDist->totalSize); + tbufWriteUint32(bw, pDist->numOfRowsInMemTable); + tbufWriteUint64(bw, taosArrayGetSize(pDist->dataBlockInfos)); + + // compress the binary string + char* p = TARRAY_GET_START(pDist->dataBlockInfos); + + // compress extra bytes + size_t x = taosArrayGetSize(pDist->dataBlockInfos) * pDist->dataBlockInfos->elemSize; + char* tmp = malloc(x + 2); + + bool comp = false; + int32_t len = tsCompressString(p, (int32_t)x, 1, tmp, (int32_t)x, ONE_STAGE_COMP, NULL, 0); + if (len == -1 || len >= x) { // compress failed, do not compress this binary data + comp = false; + len = (int32_t)x; + } else { + comp = true; + } + + tbufWriteUint8(bw, comp); + tbufWriteUint32(bw, len); + if (comp) { + tbufWriteBinary(bw, tmp, len); + } else { + tbufWriteBinary(bw, p, len); + } + tfree(tmp); +} + +void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDist) { + SBufferReader br = tbufInitReader(data, len, false); + + pDist->numOfTables = tbufReadUint32(&br); + pDist->numOfFiles = tbufReadUint16(&br); + pDist->totalSize = tbufReadUint64(&br); + pDist->numOfRowsInMemTable = tbufReadUint32(&br); + int64_t numOfBlocks = tbufReadUint64(&br); + + bool comp = tbufReadUint8(&br); + uint32_t compLen = tbufReadUint32(&br); + + size_t originalLen = (size_t) (numOfBlocks*sizeof(SFileBlockInfo)); + + char* outputBuf = NULL; + if (comp) { + outputBuf = malloc(originalLen); + + size_t actualLen = compLen; + const char* compStr = tbufReadBinary(&br, &actualLen); + + int32_t orignalLen = tsDecompressString(compStr, compLen, 1, outputBuf, + (int32_t)originalLen , ONE_STAGE_COMP, NULL, 0); + assert(orignalLen == numOfBlocks*sizeof(SFileBlockInfo)); + } else { + outputBuf = (char*) tbufReadBinary(&br, &originalLen); + } + + pDist->dataBlockInfos = taosArrayFromList(outputBuf, (uint32_t) numOfBlocks, sizeof(SFileBlockInfo)); + if (comp) { + tfree(outputBuf); + } +} + diff --git a/src/query/src/queryMain.c b/src/query/src/queryMain.c index bc4ac06a5634be924d47551d7d651585673f639e..7eb5cc26000c8520f35add531ba7f71c6f427b0f 100644 --- a/src/query/src/queryMain.c +++ b/src/query/src/queryMain.c @@ -96,7 +96,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi } if (param.pSecExprMsg != NULL) { - if ((code = createQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->secondStageOutput, ¶m.pSecExprs, param.pSecExprMsg, param.pTagColumnInfo)) != TSDB_CODE_SUCCESS) { + if ((code = createIndirectQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->secondStageOutput, ¶m.pSecExprs, param.pSecExprMsg, param.pExprs)) != TSDB_CODE_SUCCESS) { goto _over; } } @@ -144,11 +144,11 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi goto _over; } - qDebug("qmsg:%p query on %" PRIzu " tables in one group from client", pQueryMsg, tableGroupInfo.numOfTables); + qDebug("qmsg:%p query on %u tables in one group from client", pQueryMsg, tableGroupInfo.numOfTables); } int64_t el = taosGetTimestampUs() - st; - qDebug("qmsg:%p tag filter completed, numOfTables:%" PRIzu ", elapsed time:%"PRId64"us", pQueryMsg, tableGroupInfo.numOfTables, el); + qDebug("qmsg:%p tag filter completed, numOfTables:%u, elapsed time:%"PRId64"us", pQueryMsg, tableGroupInfo.numOfTables, el); } else { assert(0); } @@ -209,6 +209,7 @@ bool qTableQuery(qinfo_t qinfo) { return false; } + pQInfo->startExecTs = taosGetTimestampSec(); if (isQueryKilled(pQInfo)) { @@ -216,9 +217,10 @@ bool qTableQuery(qinfo_t qinfo) { return doBuildResCheck(pQInfo); } - if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + if (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 0) { qDebug("QInfo:%p no table exists for query, abort", pQInfo); - setQueryStatus(pQInfo->runtimeEnv.pQuery, QUERY_COMPLETED); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); return doBuildResCheck(pQInfo); } @@ -232,26 +234,16 @@ bool qTableQuery(qinfo_t qinfo) { qDebug("QInfo:%p query task is launched", pQInfo); - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - if (onlyQueryTags(pQInfo->runtimeEnv.pQuery)) { - assert(pQInfo->runtimeEnv.pQueryHandle == NULL); - buildTagQueryResult(pQInfo); - } else if (pQInfo->runtimeEnv.stableQuery) { - stableQueryImpl(pQInfo); - } else if (pQInfo->runtimeEnv.queryBlockDist){ - buildTableBlockDistResult(pQInfo); - } else { - tableQueryImpl(pQInfo); - } + pRuntimeEnv->outputBuf = pRuntimeEnv->proot->exec(pRuntimeEnv->proot); - SQuery* pQuery = pRuntimeEnv->pQuery; if (isQueryKilled(pQInfo)) { qDebug("QInfo:%p query is killed", pQInfo); - } else if (pQuery->rec.rows == 0) { - qDebug("QInfo:%p over, %" PRIzu " tables queried, %"PRId64" rows are returned", pQInfo, pQInfo->tableqinfoGroupInfo.numOfTables, pQuery->rec.total); + } else if (GET_NUM_OF_RESULTS(pRuntimeEnv) == 0) { + qDebug("QInfo:%p over, %u tables queried, %"PRId64" rows are returned", pQInfo, pRuntimeEnv->tableqinfoGroupInfo.numOfTables, + pRuntimeEnv->resultInfo.total); } else { - qDebug("QInfo:%p query paused, %" PRId64 " rows returned, numOfTotal:%" PRId64 " rows", - pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); + qDebug("QInfo:%p query paused, %d rows returned, numOfTotal:%" PRId64 " rows", + pQInfo, GET_NUM_OF_RESULTS(pRuntimeEnv), pRuntimeEnv->resultInfo.total + GET_NUM_OF_RESULTS(pRuntimeEnv)); } return doBuildResCheck(pQInfo); @@ -279,6 +271,7 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex *buildRes = true; code = pQInfo->code; } else { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; pthread_mutex_lock(&pQInfo->lock); @@ -286,8 +279,8 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex assert(pQInfo->rspContext == NULL); if (pQInfo->dataReady == QUERY_RESULT_READY) { *buildRes = true; - qDebug("QInfo:%p retrieve result info, rowsize:%d, rows:%" PRId64 ", code:%s", pQInfo, pQuery->resultRowSize, - pQuery->rec.rows, tstrerror(pQInfo->code)); + qDebug("QInfo:%p retrieve result info, rowsize:%d, rows:%d, code:%s", pQInfo, pQuery->resultRowSize, + GET_NUM_OF_RESULTS(pRuntimeEnv), tstrerror(pQInfo->code)); } else { *buildRes = false; qDebug("QInfo:%p retrieve req set query return result after paused", pQInfo); @@ -309,12 +302,13 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co return TSDB_CODE_QRY_INVALID_QHANDLE; } - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - size_t size = getResultSize(pQInfo, &pQuery->rec.rows); + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + int32_t s = GET_NUM_OF_RESULTS(pRuntimeEnv); + size_t size = pQuery->resultRowSize * s; size += sizeof(int32_t); - size += sizeof(STableIdInfo) * taosHashGetSize(pQInfo->arrTableIdInfo); + size += sizeof(STableIdInfo) * taosHashGetSize(pRuntimeEnv->pTableRetrieveTsMap); *contLen = (int32_t)(size + sizeof(SRetrieveTableRsp)); @@ -324,27 +318,27 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co return TSDB_CODE_QRY_OUT_OF_MEMORY; } - (*pRsp)->numOfRows = htonl((int32_t)pQuery->rec.rows); + (*pRsp)->numOfRows = htonl((int32_t)s); if (pQInfo->code == TSDB_CODE_SUCCESS) { - (*pRsp)->offset = htobe64(pQuery->limit.offset); - (*pRsp)->useconds = htobe64(pRuntimeEnv->summary.elapsedTime); + (*pRsp)->offset = htobe64(pQInfo->runtimeEnv.currentOffset); + (*pRsp)->useconds = htobe64(pQInfo->summary.elapsedTime); } else { (*pRsp)->offset = 0; - (*pRsp)->useconds = htobe64(pRuntimeEnv->summary.elapsedTime); + (*pRsp)->useconds = htobe64(pQInfo->summary.elapsedTime); } (*pRsp)->precision = htons(pQuery->precision); - if (pQuery->rec.rows > 0 && pQInfo->code == TSDB_CODE_SUCCESS) { + if (GET_NUM_OF_RESULTS(&(pQInfo->runtimeEnv)) > 0 && pQInfo->code == TSDB_CODE_SUCCESS) { doDumpQueryResult(pQInfo, (*pRsp)->data); } else { - setQueryStatus(pQuery, QUERY_OVER); + setQueryStatus(pRuntimeEnv, QUERY_OVER); } pQInfo->rspContext = NULL; pQInfo->dataReady = QUERY_RESULT_NOT_READY; - if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { + if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_OVER)) { // here current thread hold the refcount, so it is safe to free tsdbQueryHandle. *continueExec = false; (*pRsp)->completed = 1; // notify no more result to client @@ -394,8 +388,7 @@ int32_t qQueryCompleted(qinfo_t qinfo) { return TSDB_CODE_QRY_INVALID_QHANDLE; } - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - return isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER); + return isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQInfo->runtimeEnv.status, QUERY_OVER); } void qDestroyQueryInfo(qinfo_t qHandle) { diff --git a/src/query/src/sql.c b/src/query/src/sql.c index 019ddcc4018e5293836d5aad91bf9b4a39c2db3e..e3290272cdd7aaa2f9abe0b95d20a48abcd35fd8 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -100,27 +100,28 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 290 +#define YYNOCODE 266 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SStrToken typedef union { int yyinit; ParseTOKENTYPE yy0; - tSQLExpr* yy70; - SArray* yy161; - TAOS_FIELD yy223; - SCreateAcctInfo yy231; - SSubclauseInfo* yy233; - SIntervalVal yy300; - SCreateDbInfo yy302; - SCreatedTableInfo yy356; - int64_t yy369; - SLimitVal yy394; - int yy452; - tSQLExprList* yy458; - SCreateTableSQL* yy518; - tVariant yy526; - SQuerySQL* yy544; + SSubclauseInfo* yy21; + TAOS_FIELD yy27; + SCreateDbInfo yy114; + tSQLExpr* yy134; + tSQLExprList* yy178; + SCreateAcctInfo yy183; + SCreateTableSQL* yy190; + SCreatedTableInfo yy192; + SArray* yy193; + SQuerySQL* yy288; + int yy312; + SIntervalVal yy392; + tVariant yy442; + SSessionWindowVal yy447; + int64_t yy473; + SLimitVal yy482; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -136,18 +137,18 @@ typedef union { #define ParseCTX_FETCH #define ParseCTX_STORE #define YYFALLBACK 1 -#define YYNSTATE 312 -#define YYNRULE 266 -#define YYNRULE_WITH_ACTION 266 -#define YYNTOKEN 216 -#define YY_MAX_SHIFT 311 -#define YY_MIN_SHIFTREDUCE 503 -#define YY_MAX_SHIFTREDUCE 768 -#define YY_ERROR_ACTION 769 -#define YY_ACCEPT_ACTION 770 -#define YY_NO_ACTION 771 -#define YY_MIN_REDUCE 772 -#define YY_MAX_REDUCE 1037 +#define YYNSTATE 319 +#define YYNRULE 268 +#define YYNRULE_WITH_ACTION 268 +#define YYNTOKEN 191 +#define YY_MAX_SHIFT 318 +#define YY_MIN_SHIFTREDUCE 511 +#define YY_MAX_SHIFTREDUCE 778 +#define YY_ERROR_ACTION 779 +#define YY_ACCEPT_ACTION 780 +#define YY_NO_ACTION 781 +#define YY_MIN_REDUCE 782 +#define YY_MAX_REDUCE 1049 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -214,260 +215,260 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (667) +#define YY_ACTTAB_COUNT (675) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 78, 552, 943, 552, 201, 309, 1, 153, 17, 553, - /* 10 */ 79, 553, 72, 48, 49, 205, 52, 53, 139, 180, - /* 20 */ 213, 42, 180, 51, 257, 56, 54, 58, 55, 1019, - /* 30 */ 31, 208, 1020, 47, 46, 134, 180, 45, 44, 43, - /* 40 */ 921, 904, 905, 29, 908, 207, 1020, 504, 505, 506, - /* 50 */ 507, 508, 509, 510, 511, 512, 513, 514, 515, 516, - /* 60 */ 517, 310, 218, 940, 230, 48, 49, 932, 52, 53, - /* 70 */ 139, 203, 213, 42, 918, 51, 257, 56, 54, 58, - /* 80 */ 55, 202, 254, 71, 76, 47, 46, 921, 275, 45, - /* 90 */ 44, 43, 48, 49, 82, 52, 53, 245, 295, 213, - /* 100 */ 42, 139, 51, 257, 56, 54, 58, 55, 770, 311, - /* 110 */ 3, 166, 47, 46, 24, 225, 45, 44, 43, 48, - /* 120 */ 50, 37, 52, 53, 831, 672, 213, 42, 909, 51, - /* 130 */ 257, 56, 54, 58, 55, 632, 972, 220, 252, 47, - /* 140 */ 46, 669, 552, 45, 44, 43, 49, 25, 52, 53, - /* 150 */ 553, 178, 213, 42, 932, 51, 257, 56, 54, 58, - /* 160 */ 55, 716, 921, 285, 284, 47, 46, 971, 240, 45, - /* 170 */ 44, 43, 23, 273, 304, 303, 272, 271, 270, 302, - /* 180 */ 269, 301, 300, 299, 268, 298, 297, 881, 915, 869, - /* 190 */ 870, 871, 872, 873, 874, 875, 876, 877, 878, 879, - /* 200 */ 880, 882, 883, 52, 53, 18, 676, 213, 42, 233, - /* 210 */ 51, 257, 56, 54, 58, 55, 237, 236, 697, 698, - /* 220 */ 47, 46, 718, 188, 45, 44, 43, 212, 729, 190, - /* 230 */ 219, 720, 305, 723, 819, 726, 117, 116, 189, 165, - /* 240 */ 212, 729, 31, 31, 720, 104, 723, 224, 726, 45, - /* 250 */ 44, 43, 295, 56, 54, 58, 55, 921, 719, 209, - /* 260 */ 210, 47, 46, 256, 31, 45, 44, 43, 23, 24, - /* 270 */ 304, 303, 209, 210, 184, 302, 37, 301, 300, 299, - /* 280 */ 828, 298, 297, 216, 217, 165, 918, 918, 889, 47, - /* 290 */ 46, 887, 888, 45, 44, 43, 890, 239, 892, 893, - /* 300 */ 891, 907, 894, 895, 196, 5, 155, 258, 917, 31, - /* 310 */ 1016, 34, 154, 86, 91, 84, 90, 226, 102, 107, - /* 320 */ 282, 281, 31, 31, 96, 106, 149, 112, 115, 105, - /* 330 */ 173, 169, 31, 139, 57, 109, 171, 168, 121, 120, - /* 340 */ 119, 118, 222, 308, 307, 126, 728, 57, 225, 656, - /* 350 */ 278, 12, 653, 918, 654, 81, 655, 919, 70, 728, - /* 360 */ 820, 727, 132, 279, 283, 165, 918, 918, 642, 37, - /* 370 */ 664, 243, 32, 287, 727, 32, 918, 242, 211, 684, - /* 380 */ 227, 228, 275, 136, 62, 688, 95, 94, 689, 61, - /* 390 */ 906, 749, 20, 730, 722, 19, 725, 19, 721, 77, - /* 400 */ 724, 65, 1015, 260, 63, 644, 732, 32, 28, 61, - /* 410 */ 1014, 263, 223, 262, 643, 277, 197, 80, 61, 66, - /* 420 */ 14, 13, 101, 100, 68, 6, 631, 198, 16, 15, - /* 430 */ 660, 658, 661, 659, 114, 113, 131, 129, 182, 183, - /* 440 */ 982, 185, 179, 186, 187, 193, 1029, 194, 192, 177, - /* 450 */ 191, 181, 920, 981, 214, 978, 977, 215, 286, 934, - /* 460 */ 133, 657, 40, 942, 949, 951, 964, 135, 150, 963, - /* 470 */ 148, 914, 151, 37, 130, 241, 152, 832, 246, 265, - /* 480 */ 683, 67, 266, 267, 38, 175, 931, 35, 276, 204, - /* 490 */ 250, 827, 1034, 64, 92, 59, 1033, 140, 141, 1031, - /* 500 */ 255, 142, 253, 143, 156, 280, 1028, 251, 144, 98, - /* 510 */ 1027, 247, 1025, 249, 157, 41, 850, 36, 33, 145, - /* 520 */ 103, 39, 176, 296, 288, 816, 108, 814, 110, 111, - /* 530 */ 812, 811, 229, 167, 809, 289, 290, 808, 807, 806, - /* 540 */ 805, 804, 803, 170, 172, 800, 798, 796, 794, 792, - /* 550 */ 174, 291, 244, 73, 74, 965, 292, 293, 294, 199, - /* 560 */ 221, 264, 306, 768, 231, 200, 195, 232, 87, 88, - /* 570 */ 767, 235, 766, 234, 238, 754, 242, 666, 69, 810, - /* 580 */ 75, 259, 8, 160, 159, 851, 158, 161, 163, 162, - /* 590 */ 122, 123, 164, 124, 2, 802, 885, 801, 793, 916, - /* 600 */ 125, 685, 4, 137, 146, 147, 206, 248, 897, 690, - /* 610 */ 138, 26, 9, 10, 731, 7, 27, 11, 733, 21, - /* 620 */ 22, 261, 83, 30, 85, 81, 595, 591, 589, 588, - /* 630 */ 587, 584, 556, 274, 89, 32, 93, 60, 634, 633, - /* 640 */ 630, 579, 577, 569, 575, 571, 97, 573, 99, 567, - /* 650 */ 565, 598, 597, 596, 594, 593, 592, 590, 586, 585, - /* 660 */ 61, 554, 127, 521, 128, 772, 519, + /* 0 */ 953, 560, 184, 560, 205, 316, 79, 17, 70, 561, + /* 10 */ 80, 561, 1031, 48, 49, 142, 52, 53, 135, 72, + /* 20 */ 217, 42, 184, 51, 264, 56, 54, 58, 55, 265, + /* 30 */ 31, 212, 1032, 47, 46, 182, 184, 45, 44, 43, + /* 40 */ 916, 914, 915, 29, 918, 211, 1032, 512, 513, 514, + /* 50 */ 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, + /* 60 */ 525, 317, 950, 209, 234, 48, 49, 31, 52, 53, + /* 70 */ 142, 207, 217, 42, 928, 51, 264, 56, 54, 58, + /* 80 */ 55, 249, 984, 71, 259, 47, 46, 917, 931, 45, + /* 90 */ 44, 43, 48, 49, 925, 52, 53, 31, 302, 217, + /* 100 */ 42, 560, 51, 264, 56, 54, 58, 55, 220, 561, + /* 110 */ 31, 928, 47, 46, 24, 229, 45, 44, 43, 48, + /* 120 */ 50, 37, 52, 53, 841, 222, 217, 42, 742, 51, + /* 130 */ 264, 56, 54, 58, 55, 261, 223, 77, 221, 47, + /* 140 */ 46, 928, 224, 45, 44, 43, 49, 282, 52, 53, + /* 150 */ 931, 285, 217, 42, 928, 51, 264, 56, 54, 58, + /* 160 */ 55, 726, 315, 314, 127, 47, 46, 931, 282, 45, + /* 170 */ 44, 43, 23, 280, 311, 310, 279, 278, 277, 309, + /* 180 */ 276, 308, 307, 306, 275, 305, 304, 891, 83, 879, + /* 190 */ 880, 881, 882, 883, 884, 885, 886, 887, 888, 889, + /* 200 */ 890, 892, 893, 52, 53, 18, 188, 217, 42, 312, + /* 210 */ 51, 264, 56, 54, 58, 55, 1028, 228, 1, 157, + /* 220 */ 47, 46, 919, 192, 45, 44, 43, 216, 739, 194, + /* 230 */ 732, 730, 735, 733, 931, 736, 118, 117, 193, 899, + /* 240 */ 216, 739, 897, 898, 730, 31, 733, 900, 736, 902, + /* 250 */ 903, 901, 226, 904, 905, 45, 44, 43, 153, 213, + /* 260 */ 214, 47, 46, 263, 1027, 45, 44, 43, 23, 24, + /* 270 */ 311, 310, 213, 214, 1026, 309, 37, 308, 307, 306, + /* 280 */ 201, 305, 304, 12, 780, 318, 286, 82, 230, 928, + /* 290 */ 142, 289, 288, 237, 56, 54, 58, 55, 243, 942, + /* 300 */ 241, 240, 47, 46, 640, 200, 45, 44, 43, 202, + /* 310 */ 5, 159, 3, 170, 206, 142, 34, 158, 87, 92, + /* 320 */ 85, 91, 31, 227, 664, 680, 284, 661, 31, 662, + /* 330 */ 942, 663, 292, 291, 57, 105, 829, 707, 708, 229, + /* 340 */ 31, 169, 302, 103, 108, 244, 738, 57, 929, 97, + /* 350 */ 107, 133, 113, 116, 106, 231, 232, 983, 37, 738, + /* 360 */ 110, 677, 737, 290, 215, 728, 928, 25, 838, 294, + /* 370 */ 177, 173, 928, 169, 186, 737, 175, 172, 122, 121, + /* 380 */ 120, 119, 78, 247, 927, 830, 672, 32, 692, 62, + /* 390 */ 169, 698, 137, 246, 699, 61, 759, 65, 20, 740, + /* 400 */ 19, 729, 731, 19, 734, 650, 684, 96, 95, 32, + /* 410 */ 63, 6, 187, 267, 652, 269, 66, 32, 61, 81, + /* 420 */ 651, 68, 28, 639, 61, 270, 14, 13, 189, 102, + /* 430 */ 101, 16, 15, 668, 183, 669, 665, 666, 190, 667, + /* 440 */ 115, 114, 132, 130, 191, 197, 198, 196, 181, 195, + /* 450 */ 185, 1041, 930, 994, 993, 134, 944, 218, 952, 990, + /* 460 */ 989, 219, 293, 40, 959, 961, 136, 140, 976, 975, + /* 470 */ 37, 245, 154, 131, 152, 924, 104, 155, 691, 250, + /* 480 */ 208, 926, 303, 148, 146, 144, 156, 842, 272, 273, + /* 490 */ 59, 274, 38, 252, 179, 35, 283, 67, 257, 262, + /* 500 */ 941, 64, 837, 1046, 93, 143, 1045, 145, 258, 1043, + /* 510 */ 160, 287, 1040, 99, 1039, 1037, 260, 161, 860, 36, + /* 520 */ 33, 39, 180, 826, 109, 824, 111, 112, 822, 821, + /* 530 */ 233, 171, 819, 818, 817, 816, 815, 814, 813, 174, + /* 540 */ 176, 810, 808, 806, 804, 802, 178, 256, 248, 73, + /* 550 */ 251, 74, 253, 254, 977, 41, 295, 296, 297, 298, + /* 560 */ 299, 300, 203, 301, 313, 225, 271, 778, 235, 236, + /* 570 */ 777, 239, 204, 88, 238, 199, 89, 776, 764, 242, + /* 580 */ 246, 674, 69, 8, 138, 266, 820, 693, 164, 123, + /* 590 */ 861, 124, 162, 163, 812, 165, 166, 168, 167, 125, + /* 600 */ 811, 895, 126, 803, 4, 2, 150, 147, 75, 149, + /* 610 */ 696, 151, 139, 210, 9, 76, 255, 907, 700, 141, + /* 620 */ 10, 84, 741, 26, 7, 27, 11, 21, 743, 22, + /* 630 */ 30, 268, 86, 603, 82, 599, 597, 596, 595, 592, + /* 640 */ 564, 281, 32, 94, 60, 90, 642, 641, 638, 587, + /* 650 */ 585, 577, 583, 579, 581, 575, 98, 573, 100, 606, + /* 660 */ 605, 604, 602, 601, 600, 598, 594, 593, 61, 562, + /* 670 */ 529, 527, 128, 782, 129, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 265, 1, 219, 1, 218, 219, 226, 227, 279, 9, - /* 10 */ 225, 9, 277, 13, 14, 239, 16, 17, 219, 279, - /* 20 */ 20, 21, 279, 23, 24, 25, 26, 27, 28, 289, - /* 30 */ 219, 288, 289, 33, 34, 219, 279, 37, 38, 39, - /* 40 */ 264, 256, 257, 258, 259, 288, 289, 45, 46, 47, + /* 0 */ 194, 1, 255, 1, 193, 194, 240, 255, 200, 9, + /* 10 */ 200, 9, 265, 13, 14, 194, 16, 17, 194, 253, + /* 20 */ 20, 21, 255, 23, 24, 25, 26, 27, 28, 15, + /* 30 */ 194, 264, 265, 33, 34, 255, 255, 37, 38, 39, + /* 40 */ 232, 231, 232, 233, 234, 264, 265, 45, 46, 47, /* 50 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 60 */ 58, 59, 239, 280, 62, 13, 14, 262, 16, 17, - /* 70 */ 219, 260, 20, 21, 263, 23, 24, 25, 26, 27, - /* 80 */ 28, 276, 283, 83, 285, 33, 34, 264, 81, 37, - /* 90 */ 38, 39, 13, 14, 225, 16, 17, 281, 85, 20, - /* 100 */ 21, 219, 23, 24, 25, 26, 27, 28, 216, 217, - /* 110 */ 222, 223, 33, 34, 108, 219, 37, 38, 39, 13, - /* 120 */ 14, 115, 16, 17, 228, 37, 20, 21, 259, 23, - /* 130 */ 24, 25, 26, 27, 28, 5, 285, 239, 287, 33, - /* 140 */ 34, 113, 1, 37, 38, 39, 14, 119, 16, 17, - /* 150 */ 9, 279, 20, 21, 262, 23, 24, 25, 26, 27, - /* 160 */ 28, 109, 264, 33, 34, 33, 34, 285, 276, 37, + /* 60 */ 58, 59, 256, 214, 62, 13, 14, 194, 16, 17, + /* 70 */ 194, 235, 20, 21, 238, 23, 24, 25, 26, 27, + /* 80 */ 28, 257, 261, 83, 263, 33, 34, 0, 239, 37, + /* 90 */ 38, 39, 13, 14, 194, 16, 17, 194, 85, 20, + /* 100 */ 21, 1, 23, 24, 25, 26, 27, 28, 235, 9, + /* 110 */ 194, 238, 33, 34, 108, 194, 37, 38, 39, 13, + /* 120 */ 14, 115, 16, 17, 203, 214, 20, 21, 114, 23, + /* 130 */ 24, 25, 26, 27, 28, 259, 236, 261, 235, 33, + /* 140 */ 34, 238, 214, 37, 38, 39, 14, 81, 16, 17, + /* 150 */ 239, 235, 20, 21, 238, 23, 24, 25, 26, 27, + /* 160 */ 28, 109, 65, 66, 67, 33, 34, 239, 81, 37, /* 170 */ 38, 39, 92, 93, 94, 95, 96, 97, 98, 99, - /* 180 */ 100, 101, 102, 103, 104, 105, 106, 238, 219, 240, - /* 190 */ 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, - /* 200 */ 251, 252, 253, 16, 17, 44, 118, 20, 21, 137, - /* 210 */ 23, 24, 25, 26, 27, 28, 144, 145, 126, 127, - /* 220 */ 33, 34, 1, 62, 37, 38, 39, 1, 2, 68, - /* 230 */ 261, 5, 239, 7, 224, 9, 75, 76, 77, 229, - /* 240 */ 1, 2, 219, 219, 5, 78, 7, 68, 9, 37, - /* 250 */ 38, 39, 85, 25, 26, 27, 28, 264, 37, 33, - /* 260 */ 34, 33, 34, 37, 219, 37, 38, 39, 92, 108, - /* 270 */ 94, 95, 33, 34, 279, 99, 115, 101, 102, 103, - /* 280 */ 224, 105, 106, 260, 260, 229, 263, 263, 238, 33, - /* 290 */ 34, 241, 242, 37, 38, 39, 246, 136, 248, 249, - /* 300 */ 250, 0, 252, 253, 143, 63, 64, 15, 263, 219, - /* 310 */ 279, 69, 70, 71, 72, 73, 74, 138, 63, 64, - /* 320 */ 141, 142, 219, 219, 69, 70, 83, 72, 73, 74, - /* 330 */ 63, 64, 219, 219, 108, 80, 69, 70, 71, 72, - /* 340 */ 73, 74, 68, 65, 66, 67, 120, 108, 219, 2, - /* 350 */ 260, 108, 5, 263, 7, 112, 9, 228, 225, 120, - /* 360 */ 224, 135, 108, 260, 260, 229, 263, 263, 109, 115, - /* 370 */ 109, 109, 113, 260, 135, 113, 263, 116, 61, 109, - /* 380 */ 33, 34, 81, 113, 113, 109, 139, 140, 109, 113, - /* 390 */ 257, 109, 113, 109, 5, 113, 7, 113, 5, 285, - /* 400 */ 7, 113, 279, 109, 133, 109, 114, 113, 108, 113, - /* 410 */ 279, 111, 138, 109, 109, 141, 279, 113, 113, 131, - /* 420 */ 139, 140, 139, 140, 108, 108, 110, 279, 139, 140, - /* 430 */ 5, 5, 7, 7, 78, 79, 63, 64, 279, 279, - /* 440 */ 255, 279, 279, 279, 279, 279, 264, 279, 279, 279, - /* 450 */ 279, 279, 264, 255, 255, 255, 255, 255, 255, 262, - /* 460 */ 219, 114, 278, 219, 219, 219, 286, 219, 219, 286, - /* 470 */ 266, 219, 219, 115, 61, 262, 219, 219, 282, 219, - /* 480 */ 120, 130, 219, 219, 219, 219, 275, 219, 219, 282, - /* 490 */ 282, 219, 219, 132, 219, 129, 219, 274, 273, 219, - /* 500 */ 124, 272, 128, 271, 219, 219, 219, 123, 270, 219, - /* 510 */ 219, 121, 219, 122, 219, 134, 219, 219, 219, 269, - /* 520 */ 91, 219, 219, 107, 90, 219, 219, 219, 219, 219, - /* 530 */ 219, 219, 219, 219, 219, 51, 87, 219, 219, 219, - /* 540 */ 219, 219, 219, 219, 219, 219, 219, 219, 219, 219, - /* 550 */ 219, 89, 220, 220, 220, 220, 55, 88, 86, 220, - /* 560 */ 220, 220, 81, 5, 146, 220, 220, 5, 225, 225, - /* 570 */ 5, 5, 5, 146, 137, 93, 116, 109, 117, 220, - /* 580 */ 113, 111, 108, 231, 235, 237, 236, 234, 233, 232, - /* 590 */ 221, 221, 230, 221, 226, 220, 254, 220, 220, 262, - /* 600 */ 221, 109, 222, 108, 268, 267, 1, 108, 254, 109, - /* 610 */ 108, 113, 125, 125, 109, 108, 113, 108, 114, 108, - /* 620 */ 108, 111, 78, 84, 83, 112, 9, 5, 5, 5, - /* 630 */ 5, 5, 82, 15, 78, 113, 140, 16, 5, 5, - /* 640 */ 109, 5, 5, 5, 5, 5, 140, 5, 140, 5, - /* 650 */ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - /* 660 */ 113, 82, 21, 61, 21, 0, 60, 290, 290, 290, - /* 670 */ 290, 290, 290, 290, 290, 290, 290, 290, 290, 290, - /* 680 */ 290, 290, 290, 290, 290, 290, 290, 290, 290, 290, - /* 690 */ 290, 290, 290, 290, 290, 290, 290, 290, 290, 290, - /* 700 */ 290, 290, 290, 290, 290, 290, 290, 290, 290, 290, - /* 710 */ 290, 290, 290, 290, 290, 290, 290, 290, 290, 290, - /* 720 */ 290, 290, 290, 290, 290, 290, 290, 290, 290, 290, - /* 730 */ 290, 290, 290, 290, 290, 290, 290, 290, 290, 290, - /* 740 */ 290, 290, 290, 290, 290, 290, 290, 290, 290, 290, - /* 750 */ 290, 290, 290, 290, 290, 290, 290, 290, 290, 290, - /* 760 */ 290, 290, 290, 290, 290, 290, 290, 290, 290, 290, - /* 770 */ 290, 290, 290, 290, 290, 290, 290, 290, 290, 290, - /* 780 */ 290, 290, 290, 290, 290, 290, 290, 290, 290, 290, - /* 790 */ 290, 290, 290, 290, 290, 290, 290, 290, 290, 290, - /* 800 */ 290, 290, 290, 290, 290, 290, 290, 290, 290, 290, - /* 810 */ 290, 290, 290, 290, 290, 290, 290, 290, 290, 290, - /* 820 */ 290, 290, 290, 290, 290, 290, 290, 290, 290, 290, - /* 830 */ 290, 290, 290, 290, 290, 290, 290, 290, 290, 290, - /* 840 */ 290, 290, 290, 290, 290, 290, 290, 290, 290, 290, - /* 850 */ 290, 290, 290, 290, 290, 290, 290, 290, 290, 290, - /* 860 */ 290, 290, 290, 290, 290, 290, 290, 290, 290, 290, - /* 870 */ 290, 290, 290, 290, 290, 290, 290, 290, 290, 290, - /* 880 */ 290, 290, 290, + /* 180 */ 100, 101, 102, 103, 104, 105, 106, 213, 200, 215, + /* 190 */ 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, + /* 200 */ 226, 227, 228, 16, 17, 44, 255, 20, 21, 214, + /* 210 */ 23, 24, 25, 26, 27, 28, 255, 68, 201, 202, + /* 220 */ 33, 34, 234, 62, 37, 38, 39, 1, 2, 68, + /* 230 */ 5, 5, 7, 7, 239, 9, 75, 76, 77, 213, + /* 240 */ 1, 2, 216, 217, 5, 194, 7, 221, 9, 223, + /* 250 */ 224, 225, 68, 227, 228, 37, 38, 39, 83, 33, + /* 260 */ 34, 33, 34, 37, 255, 37, 38, 39, 92, 108, + /* 270 */ 94, 95, 33, 34, 255, 99, 115, 101, 102, 103, + /* 280 */ 255, 105, 106, 108, 191, 192, 235, 112, 139, 238, + /* 290 */ 194, 142, 143, 138, 25, 26, 27, 28, 137, 237, + /* 300 */ 145, 146, 33, 34, 5, 144, 37, 38, 39, 255, + /* 310 */ 63, 64, 197, 198, 252, 194, 69, 70, 71, 72, + /* 320 */ 73, 74, 194, 139, 2, 37, 142, 5, 194, 7, + /* 330 */ 237, 9, 33, 34, 108, 78, 199, 127, 128, 194, + /* 340 */ 194, 204, 85, 63, 64, 252, 120, 108, 203, 69, + /* 350 */ 70, 108, 72, 73, 74, 33, 34, 261, 115, 120, + /* 360 */ 80, 113, 136, 235, 61, 1, 238, 119, 199, 235, + /* 370 */ 63, 64, 238, 204, 255, 136, 69, 70, 71, 72, + /* 380 */ 73, 74, 261, 109, 238, 199, 109, 113, 109, 113, + /* 390 */ 204, 109, 113, 116, 109, 113, 109, 113, 113, 109, + /* 400 */ 113, 37, 5, 113, 7, 109, 118, 140, 141, 113, + /* 410 */ 134, 108, 255, 109, 109, 109, 132, 113, 113, 113, + /* 420 */ 109, 108, 108, 110, 113, 111, 140, 141, 255, 140, + /* 430 */ 141, 140, 141, 5, 255, 7, 114, 5, 255, 7, + /* 440 */ 78, 79, 63, 64, 255, 255, 255, 255, 255, 255, + /* 450 */ 255, 239, 239, 230, 230, 194, 237, 230, 194, 230, + /* 460 */ 230, 230, 230, 254, 194, 194, 194, 194, 262, 262, + /* 470 */ 115, 237, 194, 61, 241, 194, 91, 194, 120, 258, + /* 480 */ 258, 237, 107, 245, 247, 249, 194, 194, 194, 194, + /* 490 */ 130, 194, 194, 258, 194, 194, 194, 131, 258, 125, + /* 500 */ 251, 133, 194, 194, 194, 250, 194, 248, 124, 194, + /* 510 */ 194, 194, 194, 194, 194, 194, 129, 194, 194, 194, + /* 520 */ 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, + /* 530 */ 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, + /* 540 */ 194, 194, 194, 194, 194, 194, 194, 123, 195, 195, + /* 550 */ 121, 195, 195, 122, 195, 135, 90, 51, 87, 89, + /* 560 */ 55, 88, 195, 86, 81, 195, 195, 5, 147, 5, + /* 570 */ 5, 5, 195, 200, 147, 195, 200, 5, 93, 138, + /* 580 */ 116, 109, 117, 108, 108, 111, 195, 109, 206, 196, + /* 590 */ 212, 196, 211, 210, 195, 209, 207, 205, 208, 196, + /* 600 */ 195, 229, 196, 195, 197, 201, 243, 246, 113, 244, + /* 610 */ 109, 242, 113, 1, 126, 108, 108, 229, 109, 108, + /* 620 */ 126, 78, 109, 113, 108, 113, 108, 108, 114, 108, + /* 630 */ 84, 111, 83, 9, 112, 5, 5, 5, 5, 5, + /* 640 */ 82, 15, 113, 141, 16, 78, 5, 5, 109, 5, + /* 650 */ 5, 5, 5, 5, 5, 5, 141, 5, 141, 5, + /* 660 */ 5, 5, 5, 5, 5, 5, 5, 5, 113, 82, + /* 670 */ 61, 60, 21, 0, 21, 266, 266, 266, 266, 266, + /* 680 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, + /* 690 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, + /* 700 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, + /* 710 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, + /* 720 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, + /* 730 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, + /* 740 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, + /* 750 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, + /* 760 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, + /* 770 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, + /* 780 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, + /* 790 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, + /* 800 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, + /* 810 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, + /* 820 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, + /* 830 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, + /* 840 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, + /* 850 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, + /* 860 */ 266, 266, 266, 266, 266, 266, }; -#define YY_SHIFT_COUNT (311) +#define YY_SHIFT_COUNT (318) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (665) +#define YY_SHIFT_MAX (673) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 161, 80, 80, 176, 176, 7, 226, 239, 141, 141, - /* 10 */ 141, 141, 141, 141, 141, 141, 141, 0, 2, 239, - /* 20 */ 347, 347, 347, 347, 6, 141, 141, 141, 141, 301, - /* 30 */ 141, 141, 141, 167, 7, 13, 13, 667, 667, 667, + /* 0 */ 161, 80, 80, 176, 176, 66, 226, 239, 100, 100, + /* 10 */ 100, 100, 100, 100, 100, 100, 100, 0, 2, 239, + /* 20 */ 322, 322, 322, 322, 6, 100, 100, 100, 100, 87, + /* 30 */ 100, 100, 100, 257, 66, 13, 13, 675, 675, 675, /* 40 */ 239, 239, 239, 239, 239, 239, 239, 239, 239, 239, /* 50 */ 239, 239, 239, 239, 239, 239, 239, 239, 239, 239, - /* 60 */ 347, 347, 130, 130, 130, 130, 130, 130, 130, 254, - /* 70 */ 141, 141, 88, 141, 141, 141, 92, 92, 28, 141, - /* 80 */ 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, - /* 90 */ 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, - /* 100 */ 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, - /* 110 */ 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, - /* 120 */ 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, - /* 130 */ 141, 141, 358, 413, 413, 413, 360, 360, 360, 413, - /* 140 */ 351, 361, 366, 376, 374, 384, 391, 390, 381, 358, - /* 150 */ 413, 413, 413, 416, 7, 7, 413, 413, 429, 434, - /* 160 */ 484, 449, 462, 501, 469, 472, 416, 413, 481, 481, - /* 170 */ 413, 481, 413, 481, 413, 667, 667, 52, 79, 106, - /* 180 */ 79, 79, 132, 187, 228, 228, 228, 228, 242, 255, - /* 190 */ 267, 256, 256, 256, 256, 179, 72, 212, 212, 243, - /* 200 */ 274, 278, 261, 262, 270, 276, 279, 282, 284, 389, - /* 210 */ 393, 221, 317, 292, 271, 288, 259, 294, 296, 304, - /* 220 */ 305, 300, 247, 281, 283, 316, 289, 425, 426, 356, - /* 230 */ 373, 558, 418, 562, 565, 427, 566, 567, 482, 437, - /* 240 */ 460, 468, 461, 470, 474, 467, 492, 495, 605, 499, - /* 250 */ 500, 502, 498, 487, 503, 488, 505, 507, 504, 509, - /* 260 */ 470, 511, 510, 512, 513, 544, 539, 541, 617, 622, - /* 270 */ 623, 624, 625, 626, 550, 618, 556, 496, 522, 522, - /* 280 */ 621, 506, 508, 522, 633, 634, 531, 522, 636, 637, - /* 290 */ 638, 639, 640, 642, 644, 645, 646, 647, 648, 649, - /* 300 */ 650, 651, 652, 653, 654, 547, 579, 641, 643, 602, - /* 310 */ 606, 665, + /* 60 */ 322, 322, 299, 299, 299, 299, 299, 299, 299, 243, + /* 70 */ 100, 100, 288, 100, 100, 100, 100, 210, 210, 248, + /* 80 */ 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + /* 90 */ 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + /* 100 */ 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + /* 110 */ 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + /* 120 */ 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + /* 130 */ 100, 100, 100, 355, 412, 412, 412, 358, 358, 358, + /* 140 */ 412, 358, 412, 366, 368, 360, 374, 387, 384, 424, + /* 150 */ 431, 429, 420, 355, 412, 412, 412, 375, 66, 66, + /* 160 */ 412, 412, 385, 466, 506, 471, 470, 505, 473, 477, + /* 170 */ 375, 412, 483, 483, 412, 483, 412, 483, 412, 675, + /* 180 */ 675, 52, 79, 106, 79, 79, 132, 187, 269, 269, + /* 190 */ 269, 269, 247, 280, 307, 228, 228, 228, 228, 149, + /* 200 */ 155, 218, 218, 175, 184, 97, 277, 274, 279, 282, + /* 210 */ 285, 287, 290, 225, 397, 364, 303, 14, 276, 284, + /* 220 */ 296, 304, 305, 306, 311, 314, 267, 286, 289, 313, + /* 230 */ 291, 428, 432, 362, 379, 562, 421, 564, 565, 427, + /* 240 */ 566, 572, 485, 441, 464, 472, 465, 474, 475, 495, + /* 250 */ 478, 476, 501, 499, 507, 612, 508, 509, 511, 510, + /* 260 */ 488, 512, 494, 513, 516, 514, 518, 474, 519, 520, + /* 270 */ 521, 522, 543, 546, 549, 624, 630, 631, 632, 633, + /* 280 */ 634, 558, 626, 567, 502, 529, 529, 628, 515, 517, + /* 290 */ 529, 641, 642, 539, 529, 644, 645, 646, 647, 648, + /* 300 */ 649, 650, 652, 654, 655, 656, 657, 658, 659, 660, + /* 310 */ 661, 662, 555, 587, 651, 653, 609, 611, 673, }; -#define YY_REDUCE_COUNT (176) -#define YY_REDUCE_MIN (-271) -#define YY_REDUCE_MAX (380) +#define YY_REDUCE_COUNT (180) +#define YY_REDUCE_MIN (-253) +#define YY_REDUCE_MAX (408) static const short yy_reduce_ofst[] = { - /* 0 */ -108, -51, -51, 50, 50, -215, -257, -243, -189, -149, - /* 10 */ -201, 23, 24, 90, 103, 104, 113, -217, -214, -260, - /* 20 */ -224, -177, -102, -7, -195, -184, -118, 114, -31, -131, - /* 30 */ -104, 129, 45, 10, 133, 56, 136, -265, -220, -112, - /* 40 */ -271, -128, -5, 31, 123, 131, 137, 148, 159, 160, - /* 50 */ 162, 163, 164, 165, 166, 168, 169, 170, 171, 172, - /* 60 */ 182, 188, 185, 198, 199, 200, 201, 202, 203, 197, - /* 70 */ 241, 244, 184, 245, 246, 248, 180, 183, 204, 249, - /* 80 */ 252, 253, 257, 258, 260, 263, 264, 265, 266, 268, - /* 90 */ 269, 272, 273, 275, 277, 280, 285, 286, 287, 290, - /* 100 */ 291, 293, 295, 297, 298, 299, 302, 303, 306, 307, - /* 110 */ 308, 309, 310, 311, 312, 313, 314, 315, 318, 319, - /* 120 */ 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, - /* 130 */ 330, 331, 213, 332, 333, 334, 196, 207, 208, 335, - /* 140 */ 211, 223, 225, 229, 232, 238, 250, 336, 338, 337, - /* 150 */ 339, 340, 341, 342, 343, 344, 345, 346, 348, 350, - /* 160 */ 349, 352, 353, 357, 355, 362, 354, 359, 369, 370, - /* 170 */ 375, 372, 377, 379, 378, 368, 380, + /* 0 */ 93, -26, -26, 26, 26, -190, -233, -219, -164, -179, + /* 10 */ -124, -127, -97, -84, 51, 128, 134, -194, -189, -253, + /* 20 */ -151, -89, -72, -5, 62, -176, 96, 121, -100, -12, + /* 30 */ -79, 145, 146, 137, -192, 169, 186, -234, 17, 115, + /* 40 */ -248, -220, -49, -39, 9, 19, 25, 54, 119, 157, + /* 50 */ 173, 179, 183, 189, 190, 191, 192, 193, 194, 195, + /* 60 */ 212, 213, 223, 224, 227, 229, 230, 231, 232, 219, + /* 70 */ 261, 264, 209, 270, 271, 272, 273, 206, 207, 233, + /* 80 */ 278, 281, 283, 292, 293, 294, 295, 297, 298, 300, + /* 90 */ 301, 302, 308, 309, 310, 312, 315, 316, 317, 318, + /* 100 */ 319, 320, 321, 323, 324, 325, 326, 327, 328, 329, + /* 110 */ 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, + /* 120 */ 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, + /* 130 */ 350, 351, 352, 234, 353, 354, 356, 221, 222, 235, + /* 140 */ 357, 240, 359, 249, 255, 236, 259, 237, 361, 238, + /* 150 */ 365, 363, 369, 244, 367, 370, 371, 372, 373, 376, + /* 160 */ 377, 380, 378, 381, 383, 382, 386, 389, 390, 392, + /* 170 */ 388, 391, 393, 395, 399, 403, 405, 406, 408, 404, + /* 180 */ 407, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 769, 884, 829, 896, 817, 826, 1022, 1022, 769, 769, - /* 10 */ 769, 769, 769, 769, 769, 769, 769, 944, 789, 1022, - /* 20 */ 769, 769, 769, 769, 769, 769, 769, 769, 769, 826, - /* 30 */ 769, 769, 769, 833, 826, 833, 833, 939, 868, 886, - /* 40 */ 769, 769, 769, 769, 769, 769, 769, 769, 769, 769, - /* 50 */ 769, 769, 769, 769, 769, 769, 769, 769, 769, 769, - /* 60 */ 769, 769, 769, 769, 769, 769, 769, 769, 769, 769, - /* 70 */ 769, 769, 946, 948, 950, 769, 968, 968, 937, 769, - /* 80 */ 769, 769, 769, 769, 769, 769, 769, 769, 769, 769, - /* 90 */ 769, 769, 769, 769, 769, 769, 769, 769, 769, 769, - /* 100 */ 769, 769, 769, 769, 769, 769, 769, 769, 815, 769, - /* 110 */ 813, 769, 769, 769, 769, 769, 769, 769, 769, 769, - /* 120 */ 769, 769, 769, 769, 769, 769, 799, 769, 769, 769, - /* 130 */ 769, 769, 769, 791, 791, 791, 769, 769, 769, 791, - /* 140 */ 975, 979, 973, 961, 969, 960, 956, 955, 983, 769, - /* 150 */ 791, 791, 791, 830, 826, 826, 791, 791, 849, 847, - /* 160 */ 845, 837, 843, 839, 841, 835, 818, 791, 824, 824, - /* 170 */ 791, 824, 791, 824, 791, 868, 886, 769, 984, 769, - /* 180 */ 1021, 974, 1011, 1010, 1017, 1009, 1008, 1007, 769, 769, - /* 190 */ 769, 1003, 1004, 1006, 1005, 769, 769, 1013, 1012, 769, - /* 200 */ 769, 769, 769, 769, 769, 769, 769, 769, 769, 769, - /* 210 */ 769, 769, 986, 769, 980, 976, 769, 769, 769, 769, - /* 220 */ 769, 769, 769, 769, 769, 898, 769, 769, 769, 769, - /* 230 */ 769, 769, 769, 769, 769, 769, 769, 769, 769, 769, - /* 240 */ 936, 769, 769, 769, 769, 947, 769, 769, 769, 769, - /* 250 */ 769, 769, 970, 769, 962, 769, 769, 769, 769, 769, - /* 260 */ 910, 769, 769, 769, 769, 769, 769, 769, 769, 769, - /* 270 */ 769, 769, 769, 769, 769, 769, 769, 769, 1032, 1030, - /* 280 */ 769, 769, 769, 1026, 769, 769, 769, 1024, 769, 769, - /* 290 */ 769, 769, 769, 769, 769, 769, 769, 769, 769, 769, - /* 300 */ 769, 769, 769, 769, 769, 852, 769, 797, 795, 769, - /* 310 */ 787, 769, + /* 0 */ 779, 894, 839, 906, 827, 836, 1034, 1034, 779, 779, + /* 10 */ 779, 779, 779, 779, 779, 779, 779, 954, 799, 1034, + /* 20 */ 779, 779, 779, 779, 779, 779, 779, 779, 779, 836, + /* 30 */ 779, 779, 779, 843, 836, 843, 843, 949, 878, 896, + /* 40 */ 779, 779, 779, 779, 779, 779, 779, 779, 779, 779, + /* 50 */ 779, 779, 779, 779, 779, 779, 779, 779, 779, 779, + /* 60 */ 779, 779, 779, 779, 779, 779, 779, 779, 779, 779, + /* 70 */ 779, 779, 956, 958, 960, 779, 779, 980, 980, 947, + /* 80 */ 779, 779, 779, 779, 779, 779, 779, 779, 779, 779, + /* 90 */ 779, 779, 779, 779, 779, 779, 779, 779, 779, 779, + /* 100 */ 779, 779, 779, 779, 779, 779, 779, 779, 779, 825, + /* 110 */ 779, 823, 779, 779, 779, 779, 779, 779, 779, 779, + /* 120 */ 779, 779, 779, 779, 779, 779, 779, 809, 779, 779, + /* 130 */ 779, 779, 779, 779, 801, 801, 801, 779, 779, 779, + /* 140 */ 801, 779, 801, 987, 991, 985, 973, 981, 972, 968, + /* 150 */ 966, 965, 995, 779, 801, 801, 801, 840, 836, 836, + /* 160 */ 801, 801, 859, 857, 855, 847, 853, 849, 851, 845, + /* 170 */ 828, 801, 834, 834, 801, 834, 801, 834, 801, 878, + /* 180 */ 896, 779, 996, 779, 1033, 986, 1023, 1022, 1029, 1021, + /* 190 */ 1020, 1019, 779, 779, 779, 1015, 1016, 1018, 1017, 779, + /* 200 */ 779, 1025, 1024, 779, 779, 779, 779, 779, 779, 779, + /* 210 */ 779, 779, 779, 779, 779, 779, 998, 779, 992, 988, + /* 220 */ 779, 779, 779, 779, 779, 779, 779, 779, 779, 908, + /* 230 */ 779, 779, 779, 779, 779, 779, 779, 779, 779, 779, + /* 240 */ 779, 779, 779, 779, 946, 779, 779, 779, 779, 957, + /* 250 */ 779, 779, 779, 779, 779, 779, 779, 779, 779, 982, + /* 260 */ 779, 974, 779, 779, 779, 779, 779, 920, 779, 779, + /* 270 */ 779, 779, 779, 779, 779, 779, 779, 779, 779, 779, + /* 280 */ 779, 779, 779, 779, 779, 1044, 1042, 779, 779, 779, + /* 290 */ 1038, 779, 779, 779, 1036, 779, 779, 779, 779, 779, + /* 300 */ 779, 779, 779, 779, 779, 779, 779, 779, 779, 779, + /* 310 */ 779, 779, 862, 779, 807, 805, 779, 797, 779, }; /********** End of lemon-generated parsing tables *****************************/ @@ -609,6 +610,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* FROM => nothing */ 0, /* VARIABLE => nothing */ 0, /* INTERVAL => nothing */ + 0, /* SESSION => nothing */ 0, /* FILL => nothing */ 0, /* SLIDING => nothing */ 0, /* ORDER => nothing */ @@ -665,32 +667,6 @@ static const YYCODETYPE yyFallback[] = { 1, /* STATEMENT => ID */ 1, /* TRIGGER => ID */ 1, /* VIEW => ID */ - 1, /* COUNT => ID */ - 1, /* SUM => ID */ - 1, /* AVG => ID */ - 1, /* MIN => ID */ - 1, /* MAX => ID */ - 1, /* FIRST => ID */ - 1, /* LAST => ID */ - 1, /* TOP => ID */ - 1, /* BOTTOM => ID */ - 1, /* STDDEV => ID */ - 1, /* PERCENTILE => ID */ - 1, /* APERCENTILE => ID */ - 1, /* LEASTSQUARES => ID */ - 1, /* HISTOGRAM => ID */ - 1, /* DIFF => ID */ - 1, /* SPREAD => ID */ - 1, /* TWA => ID */ - 1, /* INTERP => ID */ - 1, /* LAST_ROW => ID */ - 1, /* RATE => ID */ - 1, /* IRATE => ID */ - 1, /* SUM_RATE => ID */ - 1, /* SUM_IRATE => ID */ - 1, /* AVG_RATE => ID */ - 1, /* AVG_IRATE => ID */ - 1, /* TBID => ID */ 1, /* SEMI => ID */ 1, /* NONE => ID */ 1, /* PREV => ID */ @@ -912,174 +888,150 @@ static const char *const yyTokenName[] = { /* 119 */ "FROM", /* 120 */ "VARIABLE", /* 121 */ "INTERVAL", - /* 122 */ "FILL", - /* 123 */ "SLIDING", - /* 124 */ "ORDER", - /* 125 */ "BY", - /* 126 */ "ASC", - /* 127 */ "DESC", - /* 128 */ "GROUP", - /* 129 */ "HAVING", - /* 130 */ "LIMIT", - /* 131 */ "OFFSET", - /* 132 */ "SLIMIT", - /* 133 */ "SOFFSET", - /* 134 */ "WHERE", - /* 135 */ "NOW", - /* 136 */ "RESET", - /* 137 */ "QUERY", - /* 138 */ "ADD", - /* 139 */ "COLUMN", - /* 140 */ "TAG", - /* 141 */ "CHANGE", - /* 142 */ "SET", - /* 143 */ "KILL", - /* 144 */ "CONNECTION", - /* 145 */ "STREAM", - /* 146 */ "COLON", - /* 147 */ "ABORT", - /* 148 */ "AFTER", - /* 149 */ "ATTACH", - /* 150 */ "BEFORE", - /* 151 */ "BEGIN", - /* 152 */ "CASCADE", - /* 153 */ "CLUSTER", - /* 154 */ "CONFLICT", - /* 155 */ "COPY", - /* 156 */ "DEFERRED", - /* 157 */ "DELIMITERS", - /* 158 */ "DETACH", - /* 159 */ "EACH", - /* 160 */ "END", - /* 161 */ "EXPLAIN", - /* 162 */ "FAIL", - /* 163 */ "FOR", - /* 164 */ "IGNORE", - /* 165 */ "IMMEDIATE", - /* 166 */ "INITIALLY", - /* 167 */ "INSTEAD", - /* 168 */ "MATCH", - /* 169 */ "KEY", - /* 170 */ "OF", - /* 171 */ "RAISE", - /* 172 */ "REPLACE", - /* 173 */ "RESTRICT", - /* 174 */ "ROW", - /* 175 */ "STATEMENT", - /* 176 */ "TRIGGER", - /* 177 */ "VIEW", - /* 178 */ "COUNT", - /* 179 */ "SUM", - /* 180 */ "AVG", - /* 181 */ "MIN", - /* 182 */ "MAX", - /* 183 */ "FIRST", - /* 184 */ "LAST", - /* 185 */ "TOP", - /* 186 */ "BOTTOM", - /* 187 */ "STDDEV", - /* 188 */ "PERCENTILE", - /* 189 */ "APERCENTILE", - /* 190 */ "LEASTSQUARES", - /* 191 */ "HISTOGRAM", - /* 192 */ "DIFF", - /* 193 */ "SPREAD", - /* 194 */ "TWA", - /* 195 */ "INTERP", - /* 196 */ "LAST_ROW", - /* 197 */ "RATE", - /* 198 */ "IRATE", - /* 199 */ "SUM_RATE", - /* 200 */ "SUM_IRATE", - /* 201 */ "AVG_RATE", - /* 202 */ "AVG_IRATE", - /* 203 */ "TBID", - /* 204 */ "SEMI", - /* 205 */ "NONE", - /* 206 */ "PREV", - /* 207 */ "LINEAR", - /* 208 */ "IMPORT", - /* 209 */ "METRIC", - /* 210 */ "TBNAME", - /* 211 */ "JOIN", - /* 212 */ "METRICS", - /* 213 */ "INSERT", - /* 214 */ "INTO", - /* 215 */ "VALUES", - /* 216 */ "program", - /* 217 */ "cmd", - /* 218 */ "dbPrefix", - /* 219 */ "ids", - /* 220 */ "cpxName", - /* 221 */ "ifexists", - /* 222 */ "alter_db_optr", - /* 223 */ "alter_topic_optr", - /* 224 */ "acct_optr", - /* 225 */ "ifnotexists", - /* 226 */ "db_optr", - /* 227 */ "topic_optr", - /* 228 */ "typename", - /* 229 */ "pps", - /* 230 */ "tseries", - /* 231 */ "dbs", - /* 232 */ "streams", - /* 233 */ "storage", - /* 234 */ "qtime", - /* 235 */ "users", - /* 236 */ "conns", - /* 237 */ "state", - /* 238 */ "keep", - /* 239 */ "tagitemlist", - /* 240 */ "cache", - /* 241 */ "replica", - /* 242 */ "quorum", - /* 243 */ "days", - /* 244 */ "minrows", - /* 245 */ "maxrows", - /* 246 */ "blocks", - /* 247 */ "ctime", - /* 248 */ "wal", - /* 249 */ "fsync", - /* 250 */ "comp", - /* 251 */ "prec", - /* 252 */ "update", - /* 253 */ "cachelast", - /* 254 */ "partitions", - /* 255 */ "signed", - /* 256 */ "create_table_args", - /* 257 */ "create_stable_args", - /* 258 */ "create_table_list", - /* 259 */ "create_from_stable", - /* 260 */ "columnlist", - /* 261 */ "tagNamelist", - /* 262 */ "select", - /* 263 */ "column", - /* 264 */ "tagitem", - /* 265 */ "selcollist", - /* 266 */ "from", - /* 267 */ "where_opt", - /* 268 */ "interval_opt", - /* 269 */ "fill_opt", - /* 270 */ "sliding_opt", - /* 271 */ "groupby_opt", - /* 272 */ "orderby_opt", - /* 273 */ "having_opt", - /* 274 */ "slimit_opt", - /* 275 */ "limit_opt", - /* 276 */ "union", - /* 277 */ "sclp", - /* 278 */ "distinct", - /* 279 */ "expr", - /* 280 */ "as", - /* 281 */ "tablelist", - /* 282 */ "tmvar", - /* 283 */ "sortlist", - /* 284 */ "sortitem", - /* 285 */ "item", - /* 286 */ "sortorder", - /* 287 */ "grouplist", - /* 288 */ "exprlist", - /* 289 */ "expritem", + /* 122 */ "SESSION", + /* 123 */ "FILL", + /* 124 */ "SLIDING", + /* 125 */ "ORDER", + /* 126 */ "BY", + /* 127 */ "ASC", + /* 128 */ "DESC", + /* 129 */ "GROUP", + /* 130 */ "HAVING", + /* 131 */ "LIMIT", + /* 132 */ "OFFSET", + /* 133 */ "SLIMIT", + /* 134 */ "SOFFSET", + /* 135 */ "WHERE", + /* 136 */ "NOW", + /* 137 */ "RESET", + /* 138 */ "QUERY", + /* 139 */ "ADD", + /* 140 */ "COLUMN", + /* 141 */ "TAG", + /* 142 */ "CHANGE", + /* 143 */ "SET", + /* 144 */ "KILL", + /* 145 */ "CONNECTION", + /* 146 */ "STREAM", + /* 147 */ "COLON", + /* 148 */ "ABORT", + /* 149 */ "AFTER", + /* 150 */ "ATTACH", + /* 151 */ "BEFORE", + /* 152 */ "BEGIN", + /* 153 */ "CASCADE", + /* 154 */ "CLUSTER", + /* 155 */ "CONFLICT", + /* 156 */ "COPY", + /* 157 */ "DEFERRED", + /* 158 */ "DELIMITERS", + /* 159 */ "DETACH", + /* 160 */ "EACH", + /* 161 */ "END", + /* 162 */ "EXPLAIN", + /* 163 */ "FAIL", + /* 164 */ "FOR", + /* 165 */ "IGNORE", + /* 166 */ "IMMEDIATE", + /* 167 */ "INITIALLY", + /* 168 */ "INSTEAD", + /* 169 */ "MATCH", + /* 170 */ "KEY", + /* 171 */ "OF", + /* 172 */ "RAISE", + /* 173 */ "REPLACE", + /* 174 */ "RESTRICT", + /* 175 */ "ROW", + /* 176 */ "STATEMENT", + /* 177 */ "TRIGGER", + /* 178 */ "VIEW", + /* 179 */ "SEMI", + /* 180 */ "NONE", + /* 181 */ "PREV", + /* 182 */ "LINEAR", + /* 183 */ "IMPORT", + /* 184 */ "METRIC", + /* 185 */ "TBNAME", + /* 186 */ "JOIN", + /* 187 */ "METRICS", + /* 188 */ "INSERT", + /* 189 */ "INTO", + /* 190 */ "VALUES", + /* 191 */ "program", + /* 192 */ "cmd", + /* 193 */ "dbPrefix", + /* 194 */ "ids", + /* 195 */ "cpxName", + /* 196 */ "ifexists", + /* 197 */ "alter_db_optr", + /* 198 */ "alter_topic_optr", + /* 199 */ "acct_optr", + /* 200 */ "ifnotexists", + /* 201 */ "db_optr", + /* 202 */ "topic_optr", + /* 203 */ "typename", + /* 204 */ "pps", + /* 205 */ "tseries", + /* 206 */ "dbs", + /* 207 */ "streams", + /* 208 */ "storage", + /* 209 */ "qtime", + /* 210 */ "users", + /* 211 */ "conns", + /* 212 */ "state", + /* 213 */ "keep", + /* 214 */ "tagitemlist", + /* 215 */ "cache", + /* 216 */ "replica", + /* 217 */ "quorum", + /* 218 */ "days", + /* 219 */ "minrows", + /* 220 */ "maxrows", + /* 221 */ "blocks", + /* 222 */ "ctime", + /* 223 */ "wal", + /* 224 */ "fsync", + /* 225 */ "comp", + /* 226 */ "prec", + /* 227 */ "update", + /* 228 */ "cachelast", + /* 229 */ "partitions", + /* 230 */ "signed", + /* 231 */ "create_table_args", + /* 232 */ "create_stable_args", + /* 233 */ "create_table_list", + /* 234 */ "create_from_stable", + /* 235 */ "columnlist", + /* 236 */ "tagNamelist", + /* 237 */ "select", + /* 238 */ "column", + /* 239 */ "tagitem", + /* 240 */ "selcollist", + /* 241 */ "from", + /* 242 */ "where_opt", + /* 243 */ "interval_opt", + /* 244 */ "session_option", + /* 245 */ "fill_opt", + /* 246 */ "sliding_opt", + /* 247 */ "groupby_opt", + /* 248 */ "orderby_opt", + /* 249 */ "having_opt", + /* 250 */ "slimit_opt", + /* 251 */ "limit_opt", + /* 252 */ "union", + /* 253 */ "sclp", + /* 254 */ "distinct", + /* 255 */ "expr", + /* 256 */ "as", + /* 257 */ "tablelist", + /* 258 */ "tmvar", + /* 259 */ "sortlist", + /* 260 */ "sortitem", + /* 261 */ "item", + /* 262 */ "sortorder", + /* 263 */ "grouplist", + /* 264 */ "exprlist", + /* 265 */ "expritem", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -1246,7 +1198,7 @@ static const char *const yyRuleName[] = { /* 156 */ "tagitem ::= MINUS FLOAT", /* 157 */ "tagitem ::= PLUS INTEGER", /* 158 */ "tagitem ::= PLUS FLOAT", - /* 159 */ "select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt", + /* 159 */ "select ::= SELECT selcollist from where_opt interval_opt session_option fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt", /* 160 */ "union ::= select", /* 161 */ "union ::= LP union RP", /* 162 */ "union ::= union UNION ALL select", @@ -1271,88 +1223,90 @@ static const char *const yyRuleName[] = { /* 181 */ "interval_opt ::= INTERVAL LP tmvar RP", /* 182 */ "interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP", /* 183 */ "interval_opt ::=", - /* 184 */ "fill_opt ::=", - /* 185 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", - /* 186 */ "fill_opt ::= FILL LP ID RP", - /* 187 */ "sliding_opt ::= SLIDING LP tmvar RP", - /* 188 */ "sliding_opt ::=", - /* 189 */ "orderby_opt ::=", - /* 190 */ "orderby_opt ::= ORDER BY sortlist", - /* 191 */ "sortlist ::= sortlist COMMA item sortorder", - /* 192 */ "sortlist ::= item sortorder", - /* 193 */ "item ::= ids cpxName", - /* 194 */ "sortorder ::= ASC", - /* 195 */ "sortorder ::= DESC", - /* 196 */ "sortorder ::=", - /* 197 */ "groupby_opt ::=", - /* 198 */ "groupby_opt ::= GROUP BY grouplist", - /* 199 */ "grouplist ::= grouplist COMMA item", - /* 200 */ "grouplist ::= item", - /* 201 */ "having_opt ::=", - /* 202 */ "having_opt ::= HAVING expr", - /* 203 */ "limit_opt ::=", - /* 204 */ "limit_opt ::= LIMIT signed", - /* 205 */ "limit_opt ::= LIMIT signed OFFSET signed", - /* 206 */ "limit_opt ::= LIMIT signed COMMA signed", - /* 207 */ "slimit_opt ::=", - /* 208 */ "slimit_opt ::= SLIMIT signed", - /* 209 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", - /* 210 */ "slimit_opt ::= SLIMIT signed COMMA signed", - /* 211 */ "where_opt ::=", - /* 212 */ "where_opt ::= WHERE expr", - /* 213 */ "expr ::= LP expr RP", - /* 214 */ "expr ::= ID", - /* 215 */ "expr ::= ID DOT ID", - /* 216 */ "expr ::= ID DOT STAR", - /* 217 */ "expr ::= INTEGER", - /* 218 */ "expr ::= MINUS INTEGER", - /* 219 */ "expr ::= PLUS INTEGER", - /* 220 */ "expr ::= FLOAT", - /* 221 */ "expr ::= MINUS FLOAT", - /* 222 */ "expr ::= PLUS FLOAT", - /* 223 */ "expr ::= STRING", - /* 224 */ "expr ::= NOW", - /* 225 */ "expr ::= VARIABLE", - /* 226 */ "expr ::= BOOL", - /* 227 */ "expr ::= ID LP exprlist RP", - /* 228 */ "expr ::= ID LP STAR RP", - /* 229 */ "expr ::= expr IS NULL", - /* 230 */ "expr ::= expr IS NOT NULL", - /* 231 */ "expr ::= expr LT expr", - /* 232 */ "expr ::= expr GT expr", - /* 233 */ "expr ::= expr LE expr", - /* 234 */ "expr ::= expr GE expr", - /* 235 */ "expr ::= expr NE expr", - /* 236 */ "expr ::= expr EQ expr", - /* 237 */ "expr ::= expr BETWEEN expr AND expr", - /* 238 */ "expr ::= expr AND expr", - /* 239 */ "expr ::= expr OR expr", - /* 240 */ "expr ::= expr PLUS expr", - /* 241 */ "expr ::= expr MINUS expr", - /* 242 */ "expr ::= expr STAR expr", - /* 243 */ "expr ::= expr SLASH expr", - /* 244 */ "expr ::= expr REM expr", - /* 245 */ "expr ::= expr LIKE expr", - /* 246 */ "expr ::= expr IN LP exprlist RP", - /* 247 */ "exprlist ::= exprlist COMMA expritem", - /* 248 */ "exprlist ::= expritem", - /* 249 */ "expritem ::= expr", - /* 250 */ "expritem ::=", - /* 251 */ "cmd ::= RESET QUERY CACHE", - /* 252 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", - /* 253 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", - /* 254 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", - /* 255 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", - /* 256 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", - /* 257 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", - /* 258 */ "cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist", - /* 259 */ "cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids", - /* 260 */ "cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist", - /* 261 */ "cmd ::= ALTER STABLE ids cpxName DROP TAG ids", - /* 262 */ "cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids", - /* 263 */ "cmd ::= KILL CONNECTION INTEGER", - /* 264 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", - /* 265 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", + /* 184 */ "session_option ::=", + /* 185 */ "session_option ::= SESSION LP ids cpxName COMMA tmvar RP", + /* 186 */ "fill_opt ::=", + /* 187 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", + /* 188 */ "fill_opt ::= FILL LP ID RP", + /* 189 */ "sliding_opt ::= SLIDING LP tmvar RP", + /* 190 */ "sliding_opt ::=", + /* 191 */ "orderby_opt ::=", + /* 192 */ "orderby_opt ::= ORDER BY sortlist", + /* 193 */ "sortlist ::= sortlist COMMA item sortorder", + /* 194 */ "sortlist ::= item sortorder", + /* 195 */ "item ::= ids cpxName", + /* 196 */ "sortorder ::= ASC", + /* 197 */ "sortorder ::= DESC", + /* 198 */ "sortorder ::=", + /* 199 */ "groupby_opt ::=", + /* 200 */ "groupby_opt ::= GROUP BY grouplist", + /* 201 */ "grouplist ::= grouplist COMMA item", + /* 202 */ "grouplist ::= item", + /* 203 */ "having_opt ::=", + /* 204 */ "having_opt ::= HAVING expr", + /* 205 */ "limit_opt ::=", + /* 206 */ "limit_opt ::= LIMIT signed", + /* 207 */ "limit_opt ::= LIMIT signed OFFSET signed", + /* 208 */ "limit_opt ::= LIMIT signed COMMA signed", + /* 209 */ "slimit_opt ::=", + /* 210 */ "slimit_opt ::= SLIMIT signed", + /* 211 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", + /* 212 */ "slimit_opt ::= SLIMIT signed COMMA signed", + /* 213 */ "where_opt ::=", + /* 214 */ "where_opt ::= WHERE expr", + /* 215 */ "expr ::= LP expr RP", + /* 216 */ "expr ::= ID", + /* 217 */ "expr ::= ID DOT ID", + /* 218 */ "expr ::= ID DOT STAR", + /* 219 */ "expr ::= INTEGER", + /* 220 */ "expr ::= MINUS INTEGER", + /* 221 */ "expr ::= PLUS INTEGER", + /* 222 */ "expr ::= FLOAT", + /* 223 */ "expr ::= MINUS FLOAT", + /* 224 */ "expr ::= PLUS FLOAT", + /* 225 */ "expr ::= STRING", + /* 226 */ "expr ::= NOW", + /* 227 */ "expr ::= VARIABLE", + /* 228 */ "expr ::= BOOL", + /* 229 */ "expr ::= ID LP exprlist RP", + /* 230 */ "expr ::= ID LP STAR RP", + /* 231 */ "expr ::= expr IS NULL", + /* 232 */ "expr ::= expr IS NOT NULL", + /* 233 */ "expr ::= expr LT expr", + /* 234 */ "expr ::= expr GT expr", + /* 235 */ "expr ::= expr LE expr", + /* 236 */ "expr ::= expr GE expr", + /* 237 */ "expr ::= expr NE expr", + /* 238 */ "expr ::= expr EQ expr", + /* 239 */ "expr ::= expr BETWEEN expr AND expr", + /* 240 */ "expr ::= expr AND expr", + /* 241 */ "expr ::= expr OR expr", + /* 242 */ "expr ::= expr PLUS expr", + /* 243 */ "expr ::= expr MINUS expr", + /* 244 */ "expr ::= expr STAR expr", + /* 245 */ "expr ::= expr SLASH expr", + /* 246 */ "expr ::= expr REM expr", + /* 247 */ "expr ::= expr LIKE expr", + /* 248 */ "expr ::= expr IN LP exprlist RP", + /* 249 */ "exprlist ::= exprlist COMMA expritem", + /* 250 */ "exprlist ::= expritem", + /* 251 */ "expritem ::= expr", + /* 252 */ "expritem ::=", + /* 253 */ "cmd ::= RESET QUERY CACHE", + /* 254 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", + /* 255 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", + /* 256 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", + /* 257 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", + /* 258 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", + /* 259 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", + /* 260 */ "cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist", + /* 261 */ "cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids", + /* 262 */ "cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist", + /* 263 */ "cmd ::= ALTER STABLE ids cpxName DROP TAG ids", + /* 264 */ "cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids", + /* 265 */ "cmd ::= KILL CONNECTION INTEGER", + /* 266 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", + /* 267 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", }; #endif /* NDEBUG */ @@ -1478,52 +1432,52 @@ static void yy_destructor( ** inside the C code. */ /********* Begin destructor definitions ***************************************/ - case 238: /* keep */ - case 239: /* tagitemlist */ - case 260: /* columnlist */ - case 261: /* tagNamelist */ - case 269: /* fill_opt */ - case 271: /* groupby_opt */ - case 272: /* orderby_opt */ - case 283: /* sortlist */ - case 287: /* grouplist */ + case 213: /* keep */ + case 214: /* tagitemlist */ + case 235: /* columnlist */ + case 236: /* tagNamelist */ + case 245: /* fill_opt */ + case 247: /* groupby_opt */ + case 248: /* orderby_opt */ + case 259: /* sortlist */ + case 263: /* grouplist */ { -taosArrayDestroy((yypminor->yy161)); +taosArrayDestroy((yypminor->yy193)); } break; - case 258: /* create_table_list */ + case 233: /* create_table_list */ { -destroyCreateTableSql((yypminor->yy518)); +destroyCreateTableSql((yypminor->yy190)); } break; - case 262: /* select */ + case 237: /* select */ { -doDestroyQuerySql((yypminor->yy544)); +doDestroyQuerySql((yypminor->yy288)); } break; - case 265: /* selcollist */ - case 277: /* sclp */ - case 288: /* exprlist */ + case 240: /* selcollist */ + case 253: /* sclp */ + case 264: /* exprlist */ { -tSqlExprListDestroy((yypminor->yy458)); +tSqlExprListDestroy((yypminor->yy178)); } break; - case 267: /* where_opt */ - case 273: /* having_opt */ - case 279: /* expr */ - case 289: /* expritem */ + case 242: /* where_opt */ + case 249: /* having_opt */ + case 255: /* expr */ + case 265: /* expritem */ { -tSqlExprDestroy((yypminor->yy70)); +tSqlExprDestroy((yypminor->yy134)); } break; - case 276: /* union */ + case 252: /* union */ { -destroyAllSelectClause((yypminor->yy233)); +destroyAllSelectClause((yypminor->yy21)); } break; - case 284: /* sortitem */ + case 260: /* sortitem */ { -tVariantDestroy(&(yypminor->yy526)); +tVariantDestroy(&(yypminor->yy442)); } break; /********* End destructor definitions *****************************************/ @@ -1812,272 +1766,274 @@ static void yy_shift( /* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side ** of that rule */ static const YYCODETYPE yyRuleInfoLhs[] = { - 216, /* (0) program ::= cmd */ - 217, /* (1) cmd ::= SHOW DATABASES */ - 217, /* (2) cmd ::= SHOW TOPICS */ - 217, /* (3) cmd ::= SHOW FUNCTIONS */ - 217, /* (4) cmd ::= SHOW MNODES */ - 217, /* (5) cmd ::= SHOW DNODES */ - 217, /* (6) cmd ::= SHOW ACCOUNTS */ - 217, /* (7) cmd ::= SHOW USERS */ - 217, /* (8) cmd ::= SHOW MODULES */ - 217, /* (9) cmd ::= SHOW QUERIES */ - 217, /* (10) cmd ::= SHOW CONNECTIONS */ - 217, /* (11) cmd ::= SHOW STREAMS */ - 217, /* (12) cmd ::= SHOW VARIABLES */ - 217, /* (13) cmd ::= SHOW SCORES */ - 217, /* (14) cmd ::= SHOW GRANTS */ - 217, /* (15) cmd ::= SHOW VNODES */ - 217, /* (16) cmd ::= SHOW VNODES IPTOKEN */ - 218, /* (17) dbPrefix ::= */ - 218, /* (18) dbPrefix ::= ids DOT */ - 220, /* (19) cpxName ::= */ - 220, /* (20) cpxName ::= DOT ids */ - 217, /* (21) cmd ::= SHOW CREATE TABLE ids cpxName */ - 217, /* (22) cmd ::= SHOW CREATE DATABASE ids */ - 217, /* (23) cmd ::= SHOW dbPrefix TABLES */ - 217, /* (24) cmd ::= SHOW dbPrefix TABLES LIKE ids */ - 217, /* (25) cmd ::= SHOW dbPrefix STABLES */ - 217, /* (26) cmd ::= SHOW dbPrefix STABLES LIKE ids */ - 217, /* (27) cmd ::= SHOW dbPrefix VGROUPS */ - 217, /* (28) cmd ::= SHOW dbPrefix VGROUPS ids */ - 217, /* (29) cmd ::= DROP TABLE ifexists ids cpxName */ - 217, /* (30) cmd ::= DROP STABLE ifexists ids cpxName */ - 217, /* (31) cmd ::= DROP DATABASE ifexists ids */ - 217, /* (32) cmd ::= DROP TOPIC ifexists ids */ - 217, /* (33) cmd ::= DROP FUNCTION ids */ - 217, /* (34) cmd ::= DROP DNODE ids */ - 217, /* (35) cmd ::= DROP USER ids */ - 217, /* (36) cmd ::= DROP ACCOUNT ids */ - 217, /* (37) cmd ::= USE ids */ - 217, /* (38) cmd ::= DESCRIBE ids cpxName */ - 217, /* (39) cmd ::= ALTER USER ids PASS ids */ - 217, /* (40) cmd ::= ALTER USER ids PRIVILEGE ids */ - 217, /* (41) cmd ::= ALTER DNODE ids ids */ - 217, /* (42) cmd ::= ALTER DNODE ids ids ids */ - 217, /* (43) cmd ::= ALTER LOCAL ids */ - 217, /* (44) cmd ::= ALTER LOCAL ids ids */ - 217, /* (45) cmd ::= ALTER DATABASE ids alter_db_optr */ - 217, /* (46) cmd ::= ALTER TOPIC ids alter_topic_optr */ - 217, /* (47) cmd ::= ALTER ACCOUNT ids acct_optr */ - 217, /* (48) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ - 219, /* (49) ids ::= ID */ - 219, /* (50) ids ::= STRING */ - 221, /* (51) ifexists ::= IF EXISTS */ - 221, /* (52) ifexists ::= */ - 225, /* (53) ifnotexists ::= IF NOT EXISTS */ - 225, /* (54) ifnotexists ::= */ - 217, /* (55) cmd ::= CREATE DNODE ids */ - 217, /* (56) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ - 217, /* (57) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ - 217, /* (58) cmd ::= CREATE TOPIC ifnotexists ids topic_optr */ - 217, /* (59) cmd ::= CREATE FUNCTION ids AS ids OUTPUTTYPE typename */ - 217, /* (60) cmd ::= CREATE USER ids PASS ids */ - 229, /* (61) pps ::= */ - 229, /* (62) pps ::= PPS INTEGER */ - 230, /* (63) tseries ::= */ - 230, /* (64) tseries ::= TSERIES INTEGER */ - 231, /* (65) dbs ::= */ - 231, /* (66) dbs ::= DBS INTEGER */ - 232, /* (67) streams ::= */ - 232, /* (68) streams ::= STREAMS INTEGER */ - 233, /* (69) storage ::= */ - 233, /* (70) storage ::= STORAGE INTEGER */ - 234, /* (71) qtime ::= */ - 234, /* (72) qtime ::= QTIME INTEGER */ - 235, /* (73) users ::= */ - 235, /* (74) users ::= USERS INTEGER */ - 236, /* (75) conns ::= */ - 236, /* (76) conns ::= CONNS INTEGER */ - 237, /* (77) state ::= */ - 237, /* (78) state ::= STATE ids */ - 224, /* (79) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ - 238, /* (80) keep ::= KEEP tagitemlist */ - 240, /* (81) cache ::= CACHE INTEGER */ - 241, /* (82) replica ::= REPLICA INTEGER */ - 242, /* (83) quorum ::= QUORUM INTEGER */ - 243, /* (84) days ::= DAYS INTEGER */ - 244, /* (85) minrows ::= MINROWS INTEGER */ - 245, /* (86) maxrows ::= MAXROWS INTEGER */ - 246, /* (87) blocks ::= BLOCKS INTEGER */ - 247, /* (88) ctime ::= CTIME INTEGER */ - 248, /* (89) wal ::= WAL INTEGER */ - 249, /* (90) fsync ::= FSYNC INTEGER */ - 250, /* (91) comp ::= COMP INTEGER */ - 251, /* (92) prec ::= PRECISION STRING */ - 252, /* (93) update ::= UPDATE INTEGER */ - 253, /* (94) cachelast ::= CACHELAST INTEGER */ - 254, /* (95) partitions ::= PARTITIONS INTEGER */ - 226, /* (96) db_optr ::= */ - 226, /* (97) db_optr ::= db_optr cache */ - 226, /* (98) db_optr ::= db_optr replica */ - 226, /* (99) db_optr ::= db_optr quorum */ - 226, /* (100) db_optr ::= db_optr days */ - 226, /* (101) db_optr ::= db_optr minrows */ - 226, /* (102) db_optr ::= db_optr maxrows */ - 226, /* (103) db_optr ::= db_optr blocks */ - 226, /* (104) db_optr ::= db_optr ctime */ - 226, /* (105) db_optr ::= db_optr wal */ - 226, /* (106) db_optr ::= db_optr fsync */ - 226, /* (107) db_optr ::= db_optr comp */ - 226, /* (108) db_optr ::= db_optr prec */ - 226, /* (109) db_optr ::= db_optr keep */ - 226, /* (110) db_optr ::= db_optr update */ - 226, /* (111) db_optr ::= db_optr cachelast */ - 227, /* (112) topic_optr ::= db_optr */ - 227, /* (113) topic_optr ::= topic_optr partitions */ - 222, /* (114) alter_db_optr ::= */ - 222, /* (115) alter_db_optr ::= alter_db_optr replica */ - 222, /* (116) alter_db_optr ::= alter_db_optr quorum */ - 222, /* (117) alter_db_optr ::= alter_db_optr keep */ - 222, /* (118) alter_db_optr ::= alter_db_optr blocks */ - 222, /* (119) alter_db_optr ::= alter_db_optr comp */ - 222, /* (120) alter_db_optr ::= alter_db_optr wal */ - 222, /* (121) alter_db_optr ::= alter_db_optr fsync */ - 222, /* (122) alter_db_optr ::= alter_db_optr update */ - 222, /* (123) alter_db_optr ::= alter_db_optr cachelast */ - 223, /* (124) alter_topic_optr ::= alter_db_optr */ - 223, /* (125) alter_topic_optr ::= alter_topic_optr partitions */ - 228, /* (126) typename ::= ids */ - 228, /* (127) typename ::= ids LP signed RP */ - 228, /* (128) typename ::= ids UNSIGNED */ - 255, /* (129) signed ::= INTEGER */ - 255, /* (130) signed ::= PLUS INTEGER */ - 255, /* (131) signed ::= MINUS INTEGER */ - 217, /* (132) cmd ::= CREATE TABLE create_table_args */ - 217, /* (133) cmd ::= CREATE TABLE create_stable_args */ - 217, /* (134) cmd ::= CREATE STABLE create_stable_args */ - 217, /* (135) cmd ::= CREATE TABLE create_table_list */ - 258, /* (136) create_table_list ::= create_from_stable */ - 258, /* (137) create_table_list ::= create_table_list create_from_stable */ - 256, /* (138) create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ - 257, /* (139) create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ - 259, /* (140) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ - 259, /* (141) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist RP */ - 261, /* (142) tagNamelist ::= tagNamelist COMMA ids */ - 261, /* (143) tagNamelist ::= ids */ - 256, /* (144) create_table_args ::= ifnotexists ids cpxName AS select */ - 260, /* (145) columnlist ::= columnlist COMMA column */ - 260, /* (146) columnlist ::= column */ - 263, /* (147) column ::= ids typename */ - 239, /* (148) tagitemlist ::= tagitemlist COMMA tagitem */ - 239, /* (149) tagitemlist ::= tagitem */ - 264, /* (150) tagitem ::= INTEGER */ - 264, /* (151) tagitem ::= FLOAT */ - 264, /* (152) tagitem ::= STRING */ - 264, /* (153) tagitem ::= BOOL */ - 264, /* (154) tagitem ::= NULL */ - 264, /* (155) tagitem ::= MINUS INTEGER */ - 264, /* (156) tagitem ::= MINUS FLOAT */ - 264, /* (157) tagitem ::= PLUS INTEGER */ - 264, /* (158) tagitem ::= PLUS FLOAT */ - 262, /* (159) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ - 276, /* (160) union ::= select */ - 276, /* (161) union ::= LP union RP */ - 276, /* (162) union ::= union UNION ALL select */ - 276, /* (163) union ::= union UNION ALL LP select RP */ - 217, /* (164) cmd ::= union */ - 262, /* (165) select ::= SELECT selcollist */ - 277, /* (166) sclp ::= selcollist COMMA */ - 277, /* (167) sclp ::= */ - 265, /* (168) selcollist ::= sclp distinct expr as */ - 265, /* (169) selcollist ::= sclp STAR */ - 280, /* (170) as ::= AS ids */ - 280, /* (171) as ::= ids */ - 280, /* (172) as ::= */ - 278, /* (173) distinct ::= DISTINCT */ - 278, /* (174) distinct ::= */ - 266, /* (175) from ::= FROM tablelist */ - 281, /* (176) tablelist ::= ids cpxName */ - 281, /* (177) tablelist ::= ids cpxName ids */ - 281, /* (178) tablelist ::= tablelist COMMA ids cpxName */ - 281, /* (179) tablelist ::= tablelist COMMA ids cpxName ids */ - 282, /* (180) tmvar ::= VARIABLE */ - 268, /* (181) interval_opt ::= INTERVAL LP tmvar RP */ - 268, /* (182) interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ - 268, /* (183) interval_opt ::= */ - 269, /* (184) fill_opt ::= */ - 269, /* (185) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ - 269, /* (186) fill_opt ::= FILL LP ID RP */ - 270, /* (187) sliding_opt ::= SLIDING LP tmvar RP */ - 270, /* (188) sliding_opt ::= */ - 272, /* (189) orderby_opt ::= */ - 272, /* (190) orderby_opt ::= ORDER BY sortlist */ - 283, /* (191) sortlist ::= sortlist COMMA item sortorder */ - 283, /* (192) sortlist ::= item sortorder */ - 285, /* (193) item ::= ids cpxName */ - 286, /* (194) sortorder ::= ASC */ - 286, /* (195) sortorder ::= DESC */ - 286, /* (196) sortorder ::= */ - 271, /* (197) groupby_opt ::= */ - 271, /* (198) groupby_opt ::= GROUP BY grouplist */ - 287, /* (199) grouplist ::= grouplist COMMA item */ - 287, /* (200) grouplist ::= item */ - 273, /* (201) having_opt ::= */ - 273, /* (202) having_opt ::= HAVING expr */ - 275, /* (203) limit_opt ::= */ - 275, /* (204) limit_opt ::= LIMIT signed */ - 275, /* (205) limit_opt ::= LIMIT signed OFFSET signed */ - 275, /* (206) limit_opt ::= LIMIT signed COMMA signed */ - 274, /* (207) slimit_opt ::= */ - 274, /* (208) slimit_opt ::= SLIMIT signed */ - 274, /* (209) slimit_opt ::= SLIMIT signed SOFFSET signed */ - 274, /* (210) slimit_opt ::= SLIMIT signed COMMA signed */ - 267, /* (211) where_opt ::= */ - 267, /* (212) where_opt ::= WHERE expr */ - 279, /* (213) expr ::= LP expr RP */ - 279, /* (214) expr ::= ID */ - 279, /* (215) expr ::= ID DOT ID */ - 279, /* (216) expr ::= ID DOT STAR */ - 279, /* (217) expr ::= INTEGER */ - 279, /* (218) expr ::= MINUS INTEGER */ - 279, /* (219) expr ::= PLUS INTEGER */ - 279, /* (220) expr ::= FLOAT */ - 279, /* (221) expr ::= MINUS FLOAT */ - 279, /* (222) expr ::= PLUS FLOAT */ - 279, /* (223) expr ::= STRING */ - 279, /* (224) expr ::= NOW */ - 279, /* (225) expr ::= VARIABLE */ - 279, /* (226) expr ::= BOOL */ - 279, /* (227) expr ::= ID LP exprlist RP */ - 279, /* (228) expr ::= ID LP STAR RP */ - 279, /* (229) expr ::= expr IS NULL */ - 279, /* (230) expr ::= expr IS NOT NULL */ - 279, /* (231) expr ::= expr LT expr */ - 279, /* (232) expr ::= expr GT expr */ - 279, /* (233) expr ::= expr LE expr */ - 279, /* (234) expr ::= expr GE expr */ - 279, /* (235) expr ::= expr NE expr */ - 279, /* (236) expr ::= expr EQ expr */ - 279, /* (237) expr ::= expr BETWEEN expr AND expr */ - 279, /* (238) expr ::= expr AND expr */ - 279, /* (239) expr ::= expr OR expr */ - 279, /* (240) expr ::= expr PLUS expr */ - 279, /* (241) expr ::= expr MINUS expr */ - 279, /* (242) expr ::= expr STAR expr */ - 279, /* (243) expr ::= expr SLASH expr */ - 279, /* (244) expr ::= expr REM expr */ - 279, /* (245) expr ::= expr LIKE expr */ - 279, /* (246) expr ::= expr IN LP exprlist RP */ - 288, /* (247) exprlist ::= exprlist COMMA expritem */ - 288, /* (248) exprlist ::= expritem */ - 289, /* (249) expritem ::= expr */ - 289, /* (250) expritem ::= */ - 217, /* (251) cmd ::= RESET QUERY CACHE */ - 217, /* (252) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ - 217, /* (253) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ - 217, /* (254) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ - 217, /* (255) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ - 217, /* (256) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ - 217, /* (257) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ - 217, /* (258) cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ - 217, /* (259) cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ - 217, /* (260) cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ - 217, /* (261) cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ - 217, /* (262) cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ - 217, /* (263) cmd ::= KILL CONNECTION INTEGER */ - 217, /* (264) cmd ::= KILL STREAM INTEGER COLON INTEGER */ - 217, /* (265) cmd ::= KILL QUERY INTEGER COLON INTEGER */ + 191, /* (0) program ::= cmd */ + 192, /* (1) cmd ::= SHOW DATABASES */ + 192, /* (2) cmd ::= SHOW TOPICS */ + 192, /* (3) cmd ::= SHOW FUNCTIONS */ + 192, /* (4) cmd ::= SHOW MNODES */ + 192, /* (5) cmd ::= SHOW DNODES */ + 192, /* (6) cmd ::= SHOW ACCOUNTS */ + 192, /* (7) cmd ::= SHOW USERS */ + 192, /* (8) cmd ::= SHOW MODULES */ + 192, /* (9) cmd ::= SHOW QUERIES */ + 192, /* (10) cmd ::= SHOW CONNECTIONS */ + 192, /* (11) cmd ::= SHOW STREAMS */ + 192, /* (12) cmd ::= SHOW VARIABLES */ + 192, /* (13) cmd ::= SHOW SCORES */ + 192, /* (14) cmd ::= SHOW GRANTS */ + 192, /* (15) cmd ::= SHOW VNODES */ + 192, /* (16) cmd ::= SHOW VNODES IPTOKEN */ + 193, /* (17) dbPrefix ::= */ + 193, /* (18) dbPrefix ::= ids DOT */ + 195, /* (19) cpxName ::= */ + 195, /* (20) cpxName ::= DOT ids */ + 192, /* (21) cmd ::= SHOW CREATE TABLE ids cpxName */ + 192, /* (22) cmd ::= SHOW CREATE DATABASE ids */ + 192, /* (23) cmd ::= SHOW dbPrefix TABLES */ + 192, /* (24) cmd ::= SHOW dbPrefix TABLES LIKE ids */ + 192, /* (25) cmd ::= SHOW dbPrefix STABLES */ + 192, /* (26) cmd ::= SHOW dbPrefix STABLES LIKE ids */ + 192, /* (27) cmd ::= SHOW dbPrefix VGROUPS */ + 192, /* (28) cmd ::= SHOW dbPrefix VGROUPS ids */ + 192, /* (29) cmd ::= DROP TABLE ifexists ids cpxName */ + 192, /* (30) cmd ::= DROP STABLE ifexists ids cpxName */ + 192, /* (31) cmd ::= DROP DATABASE ifexists ids */ + 192, /* (32) cmd ::= DROP TOPIC ifexists ids */ + 192, /* (33) cmd ::= DROP FUNCTION ids */ + 192, /* (34) cmd ::= DROP DNODE ids */ + 192, /* (35) cmd ::= DROP USER ids */ + 192, /* (36) cmd ::= DROP ACCOUNT ids */ + 192, /* (37) cmd ::= USE ids */ + 192, /* (38) cmd ::= DESCRIBE ids cpxName */ + 192, /* (39) cmd ::= ALTER USER ids PASS ids */ + 192, /* (40) cmd ::= ALTER USER ids PRIVILEGE ids */ + 192, /* (41) cmd ::= ALTER DNODE ids ids */ + 192, /* (42) cmd ::= ALTER DNODE ids ids ids */ + 192, /* (43) cmd ::= ALTER LOCAL ids */ + 192, /* (44) cmd ::= ALTER LOCAL ids ids */ + 192, /* (45) cmd ::= ALTER DATABASE ids alter_db_optr */ + 192, /* (46) cmd ::= ALTER TOPIC ids alter_topic_optr */ + 192, /* (47) cmd ::= ALTER ACCOUNT ids acct_optr */ + 192, /* (48) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ + 194, /* (49) ids ::= ID */ + 194, /* (50) ids ::= STRING */ + 196, /* (51) ifexists ::= IF EXISTS */ + 196, /* (52) ifexists ::= */ + 200, /* (53) ifnotexists ::= IF NOT EXISTS */ + 200, /* (54) ifnotexists ::= */ + 192, /* (55) cmd ::= CREATE DNODE ids */ + 192, /* (56) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ + 192, /* (57) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ + 192, /* (58) cmd ::= CREATE TOPIC ifnotexists ids topic_optr */ + 192, /* (59) cmd ::= CREATE FUNCTION ids AS ids OUTPUTTYPE typename */ + 192, /* (60) cmd ::= CREATE USER ids PASS ids */ + 204, /* (61) pps ::= */ + 204, /* (62) pps ::= PPS INTEGER */ + 205, /* (63) tseries ::= */ + 205, /* (64) tseries ::= TSERIES INTEGER */ + 206, /* (65) dbs ::= */ + 206, /* (66) dbs ::= DBS INTEGER */ + 207, /* (67) streams ::= */ + 207, /* (68) streams ::= STREAMS INTEGER */ + 208, /* (69) storage ::= */ + 208, /* (70) storage ::= STORAGE INTEGER */ + 209, /* (71) qtime ::= */ + 209, /* (72) qtime ::= QTIME INTEGER */ + 210, /* (73) users ::= */ + 210, /* (74) users ::= USERS INTEGER */ + 211, /* (75) conns ::= */ + 211, /* (76) conns ::= CONNS INTEGER */ + 212, /* (77) state ::= */ + 212, /* (78) state ::= STATE ids */ + 199, /* (79) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ + 213, /* (80) keep ::= KEEP tagitemlist */ + 215, /* (81) cache ::= CACHE INTEGER */ + 216, /* (82) replica ::= REPLICA INTEGER */ + 217, /* (83) quorum ::= QUORUM INTEGER */ + 218, /* (84) days ::= DAYS INTEGER */ + 219, /* (85) minrows ::= MINROWS INTEGER */ + 220, /* (86) maxrows ::= MAXROWS INTEGER */ + 221, /* (87) blocks ::= BLOCKS INTEGER */ + 222, /* (88) ctime ::= CTIME INTEGER */ + 223, /* (89) wal ::= WAL INTEGER */ + 224, /* (90) fsync ::= FSYNC INTEGER */ + 225, /* (91) comp ::= COMP INTEGER */ + 226, /* (92) prec ::= PRECISION STRING */ + 227, /* (93) update ::= UPDATE INTEGER */ + 228, /* (94) cachelast ::= CACHELAST INTEGER */ + 229, /* (95) partitions ::= PARTITIONS INTEGER */ + 201, /* (96) db_optr ::= */ + 201, /* (97) db_optr ::= db_optr cache */ + 201, /* (98) db_optr ::= db_optr replica */ + 201, /* (99) db_optr ::= db_optr quorum */ + 201, /* (100) db_optr ::= db_optr days */ + 201, /* (101) db_optr ::= db_optr minrows */ + 201, /* (102) db_optr ::= db_optr maxrows */ + 201, /* (103) db_optr ::= db_optr blocks */ + 201, /* (104) db_optr ::= db_optr ctime */ + 201, /* (105) db_optr ::= db_optr wal */ + 201, /* (106) db_optr ::= db_optr fsync */ + 201, /* (107) db_optr ::= db_optr comp */ + 201, /* (108) db_optr ::= db_optr prec */ + 201, /* (109) db_optr ::= db_optr keep */ + 201, /* (110) db_optr ::= db_optr update */ + 201, /* (111) db_optr ::= db_optr cachelast */ + 202, /* (112) topic_optr ::= db_optr */ + 202, /* (113) topic_optr ::= topic_optr partitions */ + 197, /* (114) alter_db_optr ::= */ + 197, /* (115) alter_db_optr ::= alter_db_optr replica */ + 197, /* (116) alter_db_optr ::= alter_db_optr quorum */ + 197, /* (117) alter_db_optr ::= alter_db_optr keep */ + 197, /* (118) alter_db_optr ::= alter_db_optr blocks */ + 197, /* (119) alter_db_optr ::= alter_db_optr comp */ + 197, /* (120) alter_db_optr ::= alter_db_optr wal */ + 197, /* (121) alter_db_optr ::= alter_db_optr fsync */ + 197, /* (122) alter_db_optr ::= alter_db_optr update */ + 197, /* (123) alter_db_optr ::= alter_db_optr cachelast */ + 198, /* (124) alter_topic_optr ::= alter_db_optr */ + 198, /* (125) alter_topic_optr ::= alter_topic_optr partitions */ + 203, /* (126) typename ::= ids */ + 203, /* (127) typename ::= ids LP signed RP */ + 203, /* (128) typename ::= ids UNSIGNED */ + 230, /* (129) signed ::= INTEGER */ + 230, /* (130) signed ::= PLUS INTEGER */ + 230, /* (131) signed ::= MINUS INTEGER */ + 192, /* (132) cmd ::= CREATE TABLE create_table_args */ + 192, /* (133) cmd ::= CREATE TABLE create_stable_args */ + 192, /* (134) cmd ::= CREATE STABLE create_stable_args */ + 192, /* (135) cmd ::= CREATE TABLE create_table_list */ + 233, /* (136) create_table_list ::= create_from_stable */ + 233, /* (137) create_table_list ::= create_table_list create_from_stable */ + 231, /* (138) create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ + 232, /* (139) create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ + 234, /* (140) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ + 234, /* (141) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist RP */ + 236, /* (142) tagNamelist ::= tagNamelist COMMA ids */ + 236, /* (143) tagNamelist ::= ids */ + 231, /* (144) create_table_args ::= ifnotexists ids cpxName AS select */ + 235, /* (145) columnlist ::= columnlist COMMA column */ + 235, /* (146) columnlist ::= column */ + 238, /* (147) column ::= ids typename */ + 214, /* (148) tagitemlist ::= tagitemlist COMMA tagitem */ + 214, /* (149) tagitemlist ::= tagitem */ + 239, /* (150) tagitem ::= INTEGER */ + 239, /* (151) tagitem ::= FLOAT */ + 239, /* (152) tagitem ::= STRING */ + 239, /* (153) tagitem ::= BOOL */ + 239, /* (154) tagitem ::= NULL */ + 239, /* (155) tagitem ::= MINUS INTEGER */ + 239, /* (156) tagitem ::= MINUS FLOAT */ + 239, /* (157) tagitem ::= PLUS INTEGER */ + 239, /* (158) tagitem ::= PLUS FLOAT */ + 237, /* (159) select ::= SELECT selcollist from where_opt interval_opt session_option fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ + 252, /* (160) union ::= select */ + 252, /* (161) union ::= LP union RP */ + 252, /* (162) union ::= union UNION ALL select */ + 252, /* (163) union ::= union UNION ALL LP select RP */ + 192, /* (164) cmd ::= union */ + 237, /* (165) select ::= SELECT selcollist */ + 253, /* (166) sclp ::= selcollist COMMA */ + 253, /* (167) sclp ::= */ + 240, /* (168) selcollist ::= sclp distinct expr as */ + 240, /* (169) selcollist ::= sclp STAR */ + 256, /* (170) as ::= AS ids */ + 256, /* (171) as ::= ids */ + 256, /* (172) as ::= */ + 254, /* (173) distinct ::= DISTINCT */ + 254, /* (174) distinct ::= */ + 241, /* (175) from ::= FROM tablelist */ + 257, /* (176) tablelist ::= ids cpxName */ + 257, /* (177) tablelist ::= ids cpxName ids */ + 257, /* (178) tablelist ::= tablelist COMMA ids cpxName */ + 257, /* (179) tablelist ::= tablelist COMMA ids cpxName ids */ + 258, /* (180) tmvar ::= VARIABLE */ + 243, /* (181) interval_opt ::= INTERVAL LP tmvar RP */ + 243, /* (182) interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ + 243, /* (183) interval_opt ::= */ + 244, /* (184) session_option ::= */ + 244, /* (185) session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ + 245, /* (186) fill_opt ::= */ + 245, /* (187) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + 245, /* (188) fill_opt ::= FILL LP ID RP */ + 246, /* (189) sliding_opt ::= SLIDING LP tmvar RP */ + 246, /* (190) sliding_opt ::= */ + 248, /* (191) orderby_opt ::= */ + 248, /* (192) orderby_opt ::= ORDER BY sortlist */ + 259, /* (193) sortlist ::= sortlist COMMA item sortorder */ + 259, /* (194) sortlist ::= item sortorder */ + 261, /* (195) item ::= ids cpxName */ + 262, /* (196) sortorder ::= ASC */ + 262, /* (197) sortorder ::= DESC */ + 262, /* (198) sortorder ::= */ + 247, /* (199) groupby_opt ::= */ + 247, /* (200) groupby_opt ::= GROUP BY grouplist */ + 263, /* (201) grouplist ::= grouplist COMMA item */ + 263, /* (202) grouplist ::= item */ + 249, /* (203) having_opt ::= */ + 249, /* (204) having_opt ::= HAVING expr */ + 251, /* (205) limit_opt ::= */ + 251, /* (206) limit_opt ::= LIMIT signed */ + 251, /* (207) limit_opt ::= LIMIT signed OFFSET signed */ + 251, /* (208) limit_opt ::= LIMIT signed COMMA signed */ + 250, /* (209) slimit_opt ::= */ + 250, /* (210) slimit_opt ::= SLIMIT signed */ + 250, /* (211) slimit_opt ::= SLIMIT signed SOFFSET signed */ + 250, /* (212) slimit_opt ::= SLIMIT signed COMMA signed */ + 242, /* (213) where_opt ::= */ + 242, /* (214) where_opt ::= WHERE expr */ + 255, /* (215) expr ::= LP expr RP */ + 255, /* (216) expr ::= ID */ + 255, /* (217) expr ::= ID DOT ID */ + 255, /* (218) expr ::= ID DOT STAR */ + 255, /* (219) expr ::= INTEGER */ + 255, /* (220) expr ::= MINUS INTEGER */ + 255, /* (221) expr ::= PLUS INTEGER */ + 255, /* (222) expr ::= FLOAT */ + 255, /* (223) expr ::= MINUS FLOAT */ + 255, /* (224) expr ::= PLUS FLOAT */ + 255, /* (225) expr ::= STRING */ + 255, /* (226) expr ::= NOW */ + 255, /* (227) expr ::= VARIABLE */ + 255, /* (228) expr ::= BOOL */ + 255, /* (229) expr ::= ID LP exprlist RP */ + 255, /* (230) expr ::= ID LP STAR RP */ + 255, /* (231) expr ::= expr IS NULL */ + 255, /* (232) expr ::= expr IS NOT NULL */ + 255, /* (233) expr ::= expr LT expr */ + 255, /* (234) expr ::= expr GT expr */ + 255, /* (235) expr ::= expr LE expr */ + 255, /* (236) expr ::= expr GE expr */ + 255, /* (237) expr ::= expr NE expr */ + 255, /* (238) expr ::= expr EQ expr */ + 255, /* (239) expr ::= expr BETWEEN expr AND expr */ + 255, /* (240) expr ::= expr AND expr */ + 255, /* (241) expr ::= expr OR expr */ + 255, /* (242) expr ::= expr PLUS expr */ + 255, /* (243) expr ::= expr MINUS expr */ + 255, /* (244) expr ::= expr STAR expr */ + 255, /* (245) expr ::= expr SLASH expr */ + 255, /* (246) expr ::= expr REM expr */ + 255, /* (247) expr ::= expr LIKE expr */ + 255, /* (248) expr ::= expr IN LP exprlist RP */ + 264, /* (249) exprlist ::= exprlist COMMA expritem */ + 264, /* (250) exprlist ::= expritem */ + 265, /* (251) expritem ::= expr */ + 265, /* (252) expritem ::= */ + 192, /* (253) cmd ::= RESET QUERY CACHE */ + 192, /* (254) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + 192, /* (255) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + 192, /* (256) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + 192, /* (257) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + 192, /* (258) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + 192, /* (259) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + 192, /* (260) cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ + 192, /* (261) cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ + 192, /* (262) cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ + 192, /* (263) cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ + 192, /* (264) cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ + 192, /* (265) cmd ::= KILL CONNECTION INTEGER */ + 192, /* (266) cmd ::= KILL STREAM INTEGER COLON INTEGER */ + 192, /* (267) cmd ::= KILL QUERY INTEGER COLON INTEGER */ }; /* For rule J, yyRuleInfoNRhs[J] contains the negative of the number @@ -2242,7 +2198,7 @@ static const signed char yyRuleInfoNRhs[] = { -2, /* (156) tagitem ::= MINUS FLOAT */ -2, /* (157) tagitem ::= PLUS INTEGER */ -2, /* (158) tagitem ::= PLUS FLOAT */ - -12, /* (159) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ + -13, /* (159) select ::= SELECT selcollist from where_opt interval_opt session_option fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ -1, /* (160) union ::= select */ -3, /* (161) union ::= LP union RP */ -4, /* (162) union ::= union UNION ALL select */ @@ -2267,88 +2223,90 @@ static const signed char yyRuleInfoNRhs[] = { -4, /* (181) interval_opt ::= INTERVAL LP tmvar RP */ -6, /* (182) interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ 0, /* (183) interval_opt ::= */ - 0, /* (184) fill_opt ::= */ - -6, /* (185) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ - -4, /* (186) fill_opt ::= FILL LP ID RP */ - -4, /* (187) sliding_opt ::= SLIDING LP tmvar RP */ - 0, /* (188) sliding_opt ::= */ - 0, /* (189) orderby_opt ::= */ - -3, /* (190) orderby_opt ::= ORDER BY sortlist */ - -4, /* (191) sortlist ::= sortlist COMMA item sortorder */ - -2, /* (192) sortlist ::= item sortorder */ - -2, /* (193) item ::= ids cpxName */ - -1, /* (194) sortorder ::= ASC */ - -1, /* (195) sortorder ::= DESC */ - 0, /* (196) sortorder ::= */ - 0, /* (197) groupby_opt ::= */ - -3, /* (198) groupby_opt ::= GROUP BY grouplist */ - -3, /* (199) grouplist ::= grouplist COMMA item */ - -1, /* (200) grouplist ::= item */ - 0, /* (201) having_opt ::= */ - -2, /* (202) having_opt ::= HAVING expr */ - 0, /* (203) limit_opt ::= */ - -2, /* (204) limit_opt ::= LIMIT signed */ - -4, /* (205) limit_opt ::= LIMIT signed OFFSET signed */ - -4, /* (206) limit_opt ::= LIMIT signed COMMA signed */ - 0, /* (207) slimit_opt ::= */ - -2, /* (208) slimit_opt ::= SLIMIT signed */ - -4, /* (209) slimit_opt ::= SLIMIT signed SOFFSET signed */ - -4, /* (210) slimit_opt ::= SLIMIT signed COMMA signed */ - 0, /* (211) where_opt ::= */ - -2, /* (212) where_opt ::= WHERE expr */ - -3, /* (213) expr ::= LP expr RP */ - -1, /* (214) expr ::= ID */ - -3, /* (215) expr ::= ID DOT ID */ - -3, /* (216) expr ::= ID DOT STAR */ - -1, /* (217) expr ::= INTEGER */ - -2, /* (218) expr ::= MINUS INTEGER */ - -2, /* (219) expr ::= PLUS INTEGER */ - -1, /* (220) expr ::= FLOAT */ - -2, /* (221) expr ::= MINUS FLOAT */ - -2, /* (222) expr ::= PLUS FLOAT */ - -1, /* (223) expr ::= STRING */ - -1, /* (224) expr ::= NOW */ - -1, /* (225) expr ::= VARIABLE */ - -1, /* (226) expr ::= BOOL */ - -4, /* (227) expr ::= ID LP exprlist RP */ - -4, /* (228) expr ::= ID LP STAR RP */ - -3, /* (229) expr ::= expr IS NULL */ - -4, /* (230) expr ::= expr IS NOT NULL */ - -3, /* (231) expr ::= expr LT expr */ - -3, /* (232) expr ::= expr GT expr */ - -3, /* (233) expr ::= expr LE expr */ - -3, /* (234) expr ::= expr GE expr */ - -3, /* (235) expr ::= expr NE expr */ - -3, /* (236) expr ::= expr EQ expr */ - -5, /* (237) expr ::= expr BETWEEN expr AND expr */ - -3, /* (238) expr ::= expr AND expr */ - -3, /* (239) expr ::= expr OR expr */ - -3, /* (240) expr ::= expr PLUS expr */ - -3, /* (241) expr ::= expr MINUS expr */ - -3, /* (242) expr ::= expr STAR expr */ - -3, /* (243) expr ::= expr SLASH expr */ - -3, /* (244) expr ::= expr REM expr */ - -3, /* (245) expr ::= expr LIKE expr */ - -5, /* (246) expr ::= expr IN LP exprlist RP */ - -3, /* (247) exprlist ::= exprlist COMMA expritem */ - -1, /* (248) exprlist ::= expritem */ - -1, /* (249) expritem ::= expr */ - 0, /* (250) expritem ::= */ - -3, /* (251) cmd ::= RESET QUERY CACHE */ - -7, /* (252) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ - -7, /* (253) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ - -7, /* (254) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ - -7, /* (255) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ - -8, /* (256) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ - -9, /* (257) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ - -7, /* (258) cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ - -7, /* (259) cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ - -7, /* (260) cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ - -7, /* (261) cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ - -8, /* (262) cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ - -3, /* (263) cmd ::= KILL CONNECTION INTEGER */ - -5, /* (264) cmd ::= KILL STREAM INTEGER COLON INTEGER */ - -5, /* (265) cmd ::= KILL QUERY INTEGER COLON INTEGER */ + 0, /* (184) session_option ::= */ + -7, /* (185) session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ + 0, /* (186) fill_opt ::= */ + -6, /* (187) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + -4, /* (188) fill_opt ::= FILL LP ID RP */ + -4, /* (189) sliding_opt ::= SLIDING LP tmvar RP */ + 0, /* (190) sliding_opt ::= */ + 0, /* (191) orderby_opt ::= */ + -3, /* (192) orderby_opt ::= ORDER BY sortlist */ + -4, /* (193) sortlist ::= sortlist COMMA item sortorder */ + -2, /* (194) sortlist ::= item sortorder */ + -2, /* (195) item ::= ids cpxName */ + -1, /* (196) sortorder ::= ASC */ + -1, /* (197) sortorder ::= DESC */ + 0, /* (198) sortorder ::= */ + 0, /* (199) groupby_opt ::= */ + -3, /* (200) groupby_opt ::= GROUP BY grouplist */ + -3, /* (201) grouplist ::= grouplist COMMA item */ + -1, /* (202) grouplist ::= item */ + 0, /* (203) having_opt ::= */ + -2, /* (204) having_opt ::= HAVING expr */ + 0, /* (205) limit_opt ::= */ + -2, /* (206) limit_opt ::= LIMIT signed */ + -4, /* (207) limit_opt ::= LIMIT signed OFFSET signed */ + -4, /* (208) limit_opt ::= LIMIT signed COMMA signed */ + 0, /* (209) slimit_opt ::= */ + -2, /* (210) slimit_opt ::= SLIMIT signed */ + -4, /* (211) slimit_opt ::= SLIMIT signed SOFFSET signed */ + -4, /* (212) slimit_opt ::= SLIMIT signed COMMA signed */ + 0, /* (213) where_opt ::= */ + -2, /* (214) where_opt ::= WHERE expr */ + -3, /* (215) expr ::= LP expr RP */ + -1, /* (216) expr ::= ID */ + -3, /* (217) expr ::= ID DOT ID */ + -3, /* (218) expr ::= ID DOT STAR */ + -1, /* (219) expr ::= INTEGER */ + -2, /* (220) expr ::= MINUS INTEGER */ + -2, /* (221) expr ::= PLUS INTEGER */ + -1, /* (222) expr ::= FLOAT */ + -2, /* (223) expr ::= MINUS FLOAT */ + -2, /* (224) expr ::= PLUS FLOAT */ + -1, /* (225) expr ::= STRING */ + -1, /* (226) expr ::= NOW */ + -1, /* (227) expr ::= VARIABLE */ + -1, /* (228) expr ::= BOOL */ + -4, /* (229) expr ::= ID LP exprlist RP */ + -4, /* (230) expr ::= ID LP STAR RP */ + -3, /* (231) expr ::= expr IS NULL */ + -4, /* (232) expr ::= expr IS NOT NULL */ + -3, /* (233) expr ::= expr LT expr */ + -3, /* (234) expr ::= expr GT expr */ + -3, /* (235) expr ::= expr LE expr */ + -3, /* (236) expr ::= expr GE expr */ + -3, /* (237) expr ::= expr NE expr */ + -3, /* (238) expr ::= expr EQ expr */ + -5, /* (239) expr ::= expr BETWEEN expr AND expr */ + -3, /* (240) expr ::= expr AND expr */ + -3, /* (241) expr ::= expr OR expr */ + -3, /* (242) expr ::= expr PLUS expr */ + -3, /* (243) expr ::= expr MINUS expr */ + -3, /* (244) expr ::= expr STAR expr */ + -3, /* (245) expr ::= expr SLASH expr */ + -3, /* (246) expr ::= expr REM expr */ + -3, /* (247) expr ::= expr LIKE expr */ + -5, /* (248) expr ::= expr IN LP exprlist RP */ + -3, /* (249) exprlist ::= exprlist COMMA expritem */ + -1, /* (250) exprlist ::= expritem */ + -1, /* (251) expritem ::= expr */ + 0, /* (252) expritem ::= */ + -3, /* (253) cmd ::= RESET QUERY CACHE */ + -7, /* (254) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + -7, /* (255) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + -7, /* (256) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + -7, /* (257) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + -8, /* (258) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + -9, /* (259) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + -7, /* (260) cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ + -7, /* (261) cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ + -7, /* (262) cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ + -7, /* (263) cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ + -8, /* (264) cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ + -3, /* (265) cmd ::= KILL CONNECTION INTEGER */ + -5, /* (266) cmd ::= KILL STREAM INTEGER COLON INTEGER */ + -5, /* (267) cmd ::= KILL QUERY INTEGER COLON INTEGER */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -2611,13 +2569,13 @@ static YYACTIONTYPE yy_reduce( break; case 45: /* cmd ::= ALTER DATABASE ids alter_db_optr */ case 46: /* cmd ::= ALTER TOPIC ids alter_topic_optr */ yytestcase(yyruleno==46); -{ SStrToken t = {0}; setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy302, &t);} +{ SStrToken t = {0}; setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy114, &t);} break; case 47: /* cmd ::= ALTER ACCOUNT ids acct_optr */ -{ setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy231);} +{ setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy183);} break; case 48: /* cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ -{ setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy231);} +{ setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy183);} break; case 49: /* ids ::= ID */ case 50: /* ids ::= STRING */ yytestcase(yyruleno==50); @@ -2639,14 +2597,14 @@ static YYACTIONTYPE yy_reduce( { setDCLSQLElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &yymsp[0].minor.yy0);} break; case 56: /* cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ -{ setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy231);} +{ setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy183);} break; case 57: /* cmd ::= CREATE DATABASE ifnotexists ids db_optr */ case 58: /* cmd ::= CREATE TOPIC ifnotexists ids topic_optr */ yytestcase(yyruleno==58); -{ setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy302, &yymsp[-2].minor.yy0);} +{ setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy114, &yymsp[-2].minor.yy0);} break; case 59: /* cmd ::= CREATE FUNCTION ids AS ids OUTPUTTYPE typename */ -{ setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &yymsp[-4].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy223);} +{ setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &yymsp[-4].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy27);} break; case 60: /* cmd ::= CREATE USER ids PASS ids */ { setCreateUserSql(pInfo, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);} @@ -2675,20 +2633,20 @@ static YYACTIONTYPE yy_reduce( break; case 79: /* acct_optr ::= pps tseries storage streams qtime dbs users conns state */ { - yylhsminor.yy231.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1; - yylhsminor.yy231.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1; - yylhsminor.yy231.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1; - yylhsminor.yy231.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1; - yylhsminor.yy231.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1; - yylhsminor.yy231.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1; - yylhsminor.yy231.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1; - yylhsminor.yy231.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1; - yylhsminor.yy231.stat = yymsp[0].minor.yy0; -} - yymsp[-8].minor.yy231 = yylhsminor.yy231; + yylhsminor.yy183.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1; + yylhsminor.yy183.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1; + yylhsminor.yy183.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1; + yylhsminor.yy183.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1; + yylhsminor.yy183.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1; + yylhsminor.yy183.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1; + yylhsminor.yy183.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1; + yylhsminor.yy183.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1; + yylhsminor.yy183.stat = yymsp[0].minor.yy0; +} + yymsp[-8].minor.yy183 = yylhsminor.yy183; break; case 80: /* keep ::= KEEP tagitemlist */ -{ yymsp[-1].minor.yy161 = yymsp[0].minor.yy161; } +{ yymsp[-1].minor.yy193 = yymsp[0].minor.yy193; } break; case 81: /* cache ::= CACHE INTEGER */ case 82: /* replica ::= REPLICA INTEGER */ yytestcase(yyruleno==82); @@ -2708,234 +2666,234 @@ static YYACTIONTYPE yy_reduce( { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } break; case 96: /* db_optr ::= */ -{setDefaultCreateDbOption(&yymsp[1].minor.yy302); yymsp[1].minor.yy302.dbType = TSDB_DB_TYPE_DEFAULT;} +{setDefaultCreateDbOption(&yymsp[1].minor.yy114); yymsp[1].minor.yy114.dbType = TSDB_DB_TYPE_DEFAULT;} break; case 97: /* db_optr ::= db_optr cache */ -{ yylhsminor.yy302 = yymsp[-1].minor.yy302; yylhsminor.yy302.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy302 = yylhsminor.yy302; +{ yylhsminor.yy114 = yymsp[-1].minor.yy114; yylhsminor.yy114.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy114 = yylhsminor.yy114; break; case 98: /* db_optr ::= db_optr replica */ case 115: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==115); -{ yylhsminor.yy302 = yymsp[-1].minor.yy302; yylhsminor.yy302.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy302 = yylhsminor.yy302; +{ yylhsminor.yy114 = yymsp[-1].minor.yy114; yylhsminor.yy114.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy114 = yylhsminor.yy114; break; case 99: /* db_optr ::= db_optr quorum */ case 116: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==116); -{ yylhsminor.yy302 = yymsp[-1].minor.yy302; yylhsminor.yy302.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy302 = yylhsminor.yy302; +{ yylhsminor.yy114 = yymsp[-1].minor.yy114; yylhsminor.yy114.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy114 = yylhsminor.yy114; break; case 100: /* db_optr ::= db_optr days */ -{ yylhsminor.yy302 = yymsp[-1].minor.yy302; yylhsminor.yy302.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy302 = yylhsminor.yy302; +{ yylhsminor.yy114 = yymsp[-1].minor.yy114; yylhsminor.yy114.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy114 = yylhsminor.yy114; break; case 101: /* db_optr ::= db_optr minrows */ -{ yylhsminor.yy302 = yymsp[-1].minor.yy302; yylhsminor.yy302.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } - yymsp[-1].minor.yy302 = yylhsminor.yy302; +{ yylhsminor.yy114 = yymsp[-1].minor.yy114; yylhsminor.yy114.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } + yymsp[-1].minor.yy114 = yylhsminor.yy114; break; case 102: /* db_optr ::= db_optr maxrows */ -{ yylhsminor.yy302 = yymsp[-1].minor.yy302; yylhsminor.yy302.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } - yymsp[-1].minor.yy302 = yylhsminor.yy302; +{ yylhsminor.yy114 = yymsp[-1].minor.yy114; yylhsminor.yy114.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } + yymsp[-1].minor.yy114 = yylhsminor.yy114; break; case 103: /* db_optr ::= db_optr blocks */ case 118: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==118); -{ yylhsminor.yy302 = yymsp[-1].minor.yy302; yylhsminor.yy302.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy302 = yylhsminor.yy302; +{ yylhsminor.yy114 = yymsp[-1].minor.yy114; yylhsminor.yy114.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy114 = yylhsminor.yy114; break; case 104: /* db_optr ::= db_optr ctime */ -{ yylhsminor.yy302 = yymsp[-1].minor.yy302; yylhsminor.yy302.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy302 = yylhsminor.yy302; +{ yylhsminor.yy114 = yymsp[-1].minor.yy114; yylhsminor.yy114.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy114 = yylhsminor.yy114; break; case 105: /* db_optr ::= db_optr wal */ case 120: /* alter_db_optr ::= alter_db_optr wal */ yytestcase(yyruleno==120); -{ yylhsminor.yy302 = yymsp[-1].minor.yy302; yylhsminor.yy302.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy302 = yylhsminor.yy302; +{ yylhsminor.yy114 = yymsp[-1].minor.yy114; yylhsminor.yy114.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy114 = yylhsminor.yy114; break; case 106: /* db_optr ::= db_optr fsync */ case 121: /* alter_db_optr ::= alter_db_optr fsync */ yytestcase(yyruleno==121); -{ yylhsminor.yy302 = yymsp[-1].minor.yy302; yylhsminor.yy302.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy302 = yylhsminor.yy302; +{ yylhsminor.yy114 = yymsp[-1].minor.yy114; yylhsminor.yy114.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy114 = yylhsminor.yy114; break; case 107: /* db_optr ::= db_optr comp */ case 119: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==119); -{ yylhsminor.yy302 = yymsp[-1].minor.yy302; yylhsminor.yy302.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy302 = yylhsminor.yy302; +{ yylhsminor.yy114 = yymsp[-1].minor.yy114; yylhsminor.yy114.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy114 = yylhsminor.yy114; break; case 108: /* db_optr ::= db_optr prec */ -{ yylhsminor.yy302 = yymsp[-1].minor.yy302; yylhsminor.yy302.precision = yymsp[0].minor.yy0; } - yymsp[-1].minor.yy302 = yylhsminor.yy302; +{ yylhsminor.yy114 = yymsp[-1].minor.yy114; yylhsminor.yy114.precision = yymsp[0].minor.yy0; } + yymsp[-1].minor.yy114 = yylhsminor.yy114; break; case 109: /* db_optr ::= db_optr keep */ case 117: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==117); -{ yylhsminor.yy302 = yymsp[-1].minor.yy302; yylhsminor.yy302.keep = yymsp[0].minor.yy161; } - yymsp[-1].minor.yy302 = yylhsminor.yy302; +{ yylhsminor.yy114 = yymsp[-1].minor.yy114; yylhsminor.yy114.keep = yymsp[0].minor.yy193; } + yymsp[-1].minor.yy114 = yylhsminor.yy114; break; case 110: /* db_optr ::= db_optr update */ case 122: /* alter_db_optr ::= alter_db_optr update */ yytestcase(yyruleno==122); -{ yylhsminor.yy302 = yymsp[-1].minor.yy302; yylhsminor.yy302.update = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy302 = yylhsminor.yy302; +{ yylhsminor.yy114 = yymsp[-1].minor.yy114; yylhsminor.yy114.update = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy114 = yylhsminor.yy114; break; case 111: /* db_optr ::= db_optr cachelast */ case 123: /* alter_db_optr ::= alter_db_optr cachelast */ yytestcase(yyruleno==123); -{ yylhsminor.yy302 = yymsp[-1].minor.yy302; yylhsminor.yy302.cachelast = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy302 = yylhsminor.yy302; +{ yylhsminor.yy114 = yymsp[-1].minor.yy114; yylhsminor.yy114.cachelast = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy114 = yylhsminor.yy114; break; case 112: /* topic_optr ::= db_optr */ case 124: /* alter_topic_optr ::= alter_db_optr */ yytestcase(yyruleno==124); -{ yylhsminor.yy302 = yymsp[0].minor.yy302; yylhsminor.yy302.dbType = TSDB_DB_TYPE_TOPIC; } - yymsp[0].minor.yy302 = yylhsminor.yy302; +{ yylhsminor.yy114 = yymsp[0].minor.yy114; yylhsminor.yy114.dbType = TSDB_DB_TYPE_TOPIC; } + yymsp[0].minor.yy114 = yylhsminor.yy114; break; case 113: /* topic_optr ::= topic_optr partitions */ case 125: /* alter_topic_optr ::= alter_topic_optr partitions */ yytestcase(yyruleno==125); -{ yylhsminor.yy302 = yymsp[-1].minor.yy302; yylhsminor.yy302.partitions = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy302 = yylhsminor.yy302; +{ yylhsminor.yy114 = yymsp[-1].minor.yy114; yylhsminor.yy114.partitions = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy114 = yylhsminor.yy114; break; case 114: /* alter_db_optr ::= */ -{ setDefaultCreateDbOption(&yymsp[1].minor.yy302); yymsp[1].minor.yy302.dbType = TSDB_DB_TYPE_DEFAULT;} +{ setDefaultCreateDbOption(&yymsp[1].minor.yy114); yymsp[1].minor.yy114.dbType = TSDB_DB_TYPE_DEFAULT;} break; case 126: /* typename ::= ids */ { yymsp[0].minor.yy0.type = 0; - tSqlSetColumnType (&yylhsminor.yy223, &yymsp[0].minor.yy0); + tSqlSetColumnType (&yylhsminor.yy27, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy223 = yylhsminor.yy223; + yymsp[0].minor.yy27 = yylhsminor.yy27; break; case 127: /* typename ::= ids LP signed RP */ { - if (yymsp[-1].minor.yy369 <= 0) { + if (yymsp[-1].minor.yy473 <= 0) { yymsp[-3].minor.yy0.type = 0; - tSqlSetColumnType(&yylhsminor.yy223, &yymsp[-3].minor.yy0); + tSqlSetColumnType(&yylhsminor.yy27, &yymsp[-3].minor.yy0); } else { - yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy369; // negative value of name length - tSqlSetColumnType(&yylhsminor.yy223, &yymsp[-3].minor.yy0); + yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy473; // negative value of name length + tSqlSetColumnType(&yylhsminor.yy27, &yymsp[-3].minor.yy0); } } - yymsp[-3].minor.yy223 = yylhsminor.yy223; + yymsp[-3].minor.yy27 = yylhsminor.yy27; break; case 128: /* typename ::= ids UNSIGNED */ { yymsp[-1].minor.yy0.type = 0; yymsp[-1].minor.yy0.n = ((yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z); - tSqlSetColumnType (&yylhsminor.yy223, &yymsp[-1].minor.yy0); + tSqlSetColumnType (&yylhsminor.yy27, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy223 = yylhsminor.yy223; + yymsp[-1].minor.yy27 = yylhsminor.yy27; break; case 129: /* signed ::= INTEGER */ -{ yylhsminor.yy369 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[0].minor.yy369 = yylhsminor.yy369; +{ yylhsminor.yy473 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[0].minor.yy473 = yylhsminor.yy473; break; case 130: /* signed ::= PLUS INTEGER */ -{ yymsp[-1].minor.yy369 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } +{ yymsp[-1].minor.yy473 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; case 131: /* signed ::= MINUS INTEGER */ -{ yymsp[-1].minor.yy369 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} +{ yymsp[-1].minor.yy473 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} break; case 135: /* cmd ::= CREATE TABLE create_table_list */ -{ pInfo->type = TSDB_SQL_CREATE_TABLE; pInfo->pCreateTableInfo = yymsp[0].minor.yy518;} +{ pInfo->type = TSDB_SQL_CREATE_TABLE; pInfo->pCreateTableInfo = yymsp[0].minor.yy190;} break; case 136: /* create_table_list ::= create_from_stable */ { SCreateTableSQL* pCreateTable = calloc(1, sizeof(SCreateTableSQL)); pCreateTable->childTableInfo = taosArrayInit(4, sizeof(SCreatedTableInfo)); - taosArrayPush(pCreateTable->childTableInfo, &yymsp[0].minor.yy356); + taosArrayPush(pCreateTable->childTableInfo, &yymsp[0].minor.yy192); pCreateTable->type = TSQL_CREATE_TABLE_FROM_STABLE; - yylhsminor.yy518 = pCreateTable; + yylhsminor.yy190 = pCreateTable; } - yymsp[0].minor.yy518 = yylhsminor.yy518; + yymsp[0].minor.yy190 = yylhsminor.yy190; break; case 137: /* create_table_list ::= create_table_list create_from_stable */ { - taosArrayPush(yymsp[-1].minor.yy518->childTableInfo, &yymsp[0].minor.yy356); - yylhsminor.yy518 = yymsp[-1].minor.yy518; + taosArrayPush(yymsp[-1].minor.yy190->childTableInfo, &yymsp[0].minor.yy192); + yylhsminor.yy190 = yymsp[-1].minor.yy190; } - yymsp[-1].minor.yy518 = yylhsminor.yy518; + yymsp[-1].minor.yy190 = yylhsminor.yy190; break; case 138: /* create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ { - yylhsminor.yy518 = tSetCreateSqlElems(yymsp[-1].minor.yy161, NULL, NULL, TSQL_CREATE_TABLE); - setSqlInfo(pInfo, yylhsminor.yy518, NULL, TSDB_SQL_CREATE_TABLE); + yylhsminor.yy190 = tSetCreateSqlElems(yymsp[-1].minor.yy193, NULL, NULL, TSQL_CREATE_TABLE); + setSqlInfo(pInfo, yylhsminor.yy190, NULL, TSDB_SQL_CREATE_TABLE); yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; setCreatedTableName(pInfo, &yymsp[-4].minor.yy0, &yymsp[-5].minor.yy0); } - yymsp[-5].minor.yy518 = yylhsminor.yy518; + yymsp[-5].minor.yy190 = yylhsminor.yy190; break; case 139: /* create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ { - yylhsminor.yy518 = tSetCreateSqlElems(yymsp[-5].minor.yy161, yymsp[-1].minor.yy161, NULL, TSQL_CREATE_STABLE); - setSqlInfo(pInfo, yylhsminor.yy518, NULL, TSDB_SQL_CREATE_TABLE); + yylhsminor.yy190 = tSetCreateSqlElems(yymsp[-5].minor.yy193, yymsp[-1].minor.yy193, NULL, TSQL_CREATE_STABLE); + setSqlInfo(pInfo, yylhsminor.yy190, NULL, TSDB_SQL_CREATE_TABLE); yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; setCreatedTableName(pInfo, &yymsp[-8].minor.yy0, &yymsp[-9].minor.yy0); } - yymsp[-9].minor.yy518 = yylhsminor.yy518; + yymsp[-9].minor.yy190 = yylhsminor.yy190; break; case 140: /* create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; - yylhsminor.yy356 = createNewChildTableInfo(&yymsp[-5].minor.yy0, NULL, yymsp[-1].minor.yy161, &yymsp[-8].minor.yy0, &yymsp[-9].minor.yy0); + yylhsminor.yy192 = createNewChildTableInfo(&yymsp[-5].minor.yy0, NULL, yymsp[-1].minor.yy193, &yymsp[-8].minor.yy0, &yymsp[-9].minor.yy0); } - yymsp[-9].minor.yy356 = yylhsminor.yy356; + yymsp[-9].minor.yy192 = yylhsminor.yy192; break; case 141: /* create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist RP */ { yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; yymsp[-11].minor.yy0.n += yymsp[-10].minor.yy0.n; - yylhsminor.yy356 = createNewChildTableInfo(&yymsp[-8].minor.yy0, yymsp[-5].minor.yy161, yymsp[-1].minor.yy161, &yymsp[-11].minor.yy0, &yymsp[-12].minor.yy0); + yylhsminor.yy192 = createNewChildTableInfo(&yymsp[-8].minor.yy0, yymsp[-5].minor.yy193, yymsp[-1].minor.yy193, &yymsp[-11].minor.yy0, &yymsp[-12].minor.yy0); } - yymsp[-12].minor.yy356 = yylhsminor.yy356; + yymsp[-12].minor.yy192 = yylhsminor.yy192; break; case 142: /* tagNamelist ::= tagNamelist COMMA ids */ -{taosArrayPush(yymsp[-2].minor.yy161, &yymsp[0].minor.yy0); yylhsminor.yy161 = yymsp[-2].minor.yy161; } - yymsp[-2].minor.yy161 = yylhsminor.yy161; +{taosArrayPush(yymsp[-2].minor.yy193, &yymsp[0].minor.yy0); yylhsminor.yy193 = yymsp[-2].minor.yy193; } + yymsp[-2].minor.yy193 = yylhsminor.yy193; break; case 143: /* tagNamelist ::= ids */ -{yylhsminor.yy161 = taosArrayInit(4, sizeof(SStrToken)); taosArrayPush(yylhsminor.yy161, &yymsp[0].minor.yy0);} - yymsp[0].minor.yy161 = yylhsminor.yy161; +{yylhsminor.yy193 = taosArrayInit(4, sizeof(SStrToken)); taosArrayPush(yylhsminor.yy193, &yymsp[0].minor.yy0);} + yymsp[0].minor.yy193 = yylhsminor.yy193; break; case 144: /* create_table_args ::= ifnotexists ids cpxName AS select */ { - yylhsminor.yy518 = tSetCreateSqlElems(NULL, NULL, yymsp[0].minor.yy544, TSQL_CREATE_STREAM); - setSqlInfo(pInfo, yylhsminor.yy518, NULL, TSDB_SQL_CREATE_TABLE); + yylhsminor.yy190 = tSetCreateSqlElems(NULL, NULL, yymsp[0].minor.yy288, TSQL_CREATE_STREAM); + setSqlInfo(pInfo, yylhsminor.yy190, NULL, TSDB_SQL_CREATE_TABLE); yymsp[-3].minor.yy0.n += yymsp[-2].minor.yy0.n; setCreatedTableName(pInfo, &yymsp[-3].minor.yy0, &yymsp[-4].minor.yy0); } - yymsp[-4].minor.yy518 = yylhsminor.yy518; + yymsp[-4].minor.yy190 = yylhsminor.yy190; break; case 145: /* columnlist ::= columnlist COMMA column */ -{taosArrayPush(yymsp[-2].minor.yy161, &yymsp[0].minor.yy223); yylhsminor.yy161 = yymsp[-2].minor.yy161; } - yymsp[-2].minor.yy161 = yylhsminor.yy161; +{taosArrayPush(yymsp[-2].minor.yy193, &yymsp[0].minor.yy27); yylhsminor.yy193 = yymsp[-2].minor.yy193; } + yymsp[-2].minor.yy193 = yylhsminor.yy193; break; case 146: /* columnlist ::= column */ -{yylhsminor.yy161 = taosArrayInit(4, sizeof(TAOS_FIELD)); taosArrayPush(yylhsminor.yy161, &yymsp[0].minor.yy223);} - yymsp[0].minor.yy161 = yylhsminor.yy161; +{yylhsminor.yy193 = taosArrayInit(4, sizeof(TAOS_FIELD)); taosArrayPush(yylhsminor.yy193, &yymsp[0].minor.yy27);} + yymsp[0].minor.yy193 = yylhsminor.yy193; break; case 147: /* column ::= ids typename */ { - tSqlSetColumnInfo(&yylhsminor.yy223, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy223); + tSqlSetColumnInfo(&yylhsminor.yy27, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy27); } - yymsp[-1].minor.yy223 = yylhsminor.yy223; + yymsp[-1].minor.yy27 = yylhsminor.yy27; break; case 148: /* tagitemlist ::= tagitemlist COMMA tagitem */ -{ yylhsminor.yy161 = tVariantListAppend(yymsp[-2].minor.yy161, &yymsp[0].minor.yy526, -1); } - yymsp[-2].minor.yy161 = yylhsminor.yy161; +{ yylhsminor.yy193 = tVariantListAppend(yymsp[-2].minor.yy193, &yymsp[0].minor.yy442, -1); } + yymsp[-2].minor.yy193 = yylhsminor.yy193; break; case 149: /* tagitemlist ::= tagitem */ -{ yylhsminor.yy161 = tVariantListAppend(NULL, &yymsp[0].minor.yy526, -1); } - yymsp[0].minor.yy161 = yylhsminor.yy161; +{ yylhsminor.yy193 = tVariantListAppend(NULL, &yymsp[0].minor.yy442, -1); } + yymsp[0].minor.yy193 = yylhsminor.yy193; break; case 150: /* tagitem ::= INTEGER */ case 151: /* tagitem ::= FLOAT */ yytestcase(yyruleno==151); case 152: /* tagitem ::= STRING */ yytestcase(yyruleno==152); case 153: /* tagitem ::= BOOL */ yytestcase(yyruleno==153); -{ toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy526, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy526 = yylhsminor.yy526; +{ toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy442, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy442 = yylhsminor.yy442; break; case 154: /* tagitem ::= NULL */ -{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy526, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy526 = yylhsminor.yy526; +{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy442, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy442 = yylhsminor.yy442; break; case 155: /* tagitem ::= MINUS INTEGER */ case 156: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==156); @@ -2945,59 +2903,59 @@ static YYACTIONTYPE yy_reduce( yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = yymsp[0].minor.yy0.type; toTSDBType(yymsp[-1].minor.yy0.type); - tVariantCreate(&yylhsminor.yy526, &yymsp[-1].minor.yy0); + tVariantCreate(&yylhsminor.yy442, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy526 = yylhsminor.yy526; + yymsp[-1].minor.yy442 = yylhsminor.yy442; break; - case 159: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ + case 159: /* select ::= SELECT selcollist from where_opt interval_opt session_option fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ { - yylhsminor.yy544 = tSetQuerySqlElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy458, yymsp[-9].minor.yy161, yymsp[-8].minor.yy70, yymsp[-4].minor.yy161, yymsp[-3].minor.yy161, &yymsp[-7].minor.yy300, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy161, &yymsp[0].minor.yy394, &yymsp[-1].minor.yy394); + yylhsminor.yy288 = tSetQuerySqlNode(&yymsp[-12].minor.yy0, yymsp[-11].minor.yy178, yymsp[-10].minor.yy193, yymsp[-9].minor.yy134, yymsp[-4].minor.yy193, yymsp[-3].minor.yy193, &yymsp[-8].minor.yy392, &yymsp[-7].minor.yy447, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy193, &yymsp[0].minor.yy482, &yymsp[-1].minor.yy482); } - yymsp[-11].minor.yy544 = yylhsminor.yy544; + yymsp[-12].minor.yy288 = yylhsminor.yy288; break; case 160: /* union ::= select */ -{ yylhsminor.yy233 = setSubclause(NULL, yymsp[0].minor.yy544); } - yymsp[0].minor.yy233 = yylhsminor.yy233; +{ yylhsminor.yy21 = setSubclause(NULL, yymsp[0].minor.yy288); } + yymsp[0].minor.yy21 = yylhsminor.yy21; break; case 161: /* union ::= LP union RP */ -{ yymsp[-2].minor.yy233 = yymsp[-1].minor.yy233; } +{ yymsp[-2].minor.yy21 = yymsp[-1].minor.yy21; } break; case 162: /* union ::= union UNION ALL select */ -{ yylhsminor.yy233 = appendSelectClause(yymsp[-3].minor.yy233, yymsp[0].minor.yy544); } - yymsp[-3].minor.yy233 = yylhsminor.yy233; +{ yylhsminor.yy21 = appendSelectClause(yymsp[-3].minor.yy21, yymsp[0].minor.yy288); } + yymsp[-3].minor.yy21 = yylhsminor.yy21; break; case 163: /* union ::= union UNION ALL LP select RP */ -{ yylhsminor.yy233 = appendSelectClause(yymsp[-5].minor.yy233, yymsp[-1].minor.yy544); } - yymsp[-5].minor.yy233 = yylhsminor.yy233; +{ yylhsminor.yy21 = appendSelectClause(yymsp[-5].minor.yy21, yymsp[-1].minor.yy288); } + yymsp[-5].minor.yy21 = yylhsminor.yy21; break; case 164: /* cmd ::= union */ -{ setSqlInfo(pInfo, yymsp[0].minor.yy233, NULL, TSDB_SQL_SELECT); } +{ setSqlInfo(pInfo, yymsp[0].minor.yy21, NULL, TSDB_SQL_SELECT); } break; case 165: /* select ::= SELECT selcollist */ { - yylhsminor.yy544 = tSetQuerySqlElems(&yymsp[-1].minor.yy0, yymsp[0].minor.yy458, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); + yylhsminor.yy288 = tSetQuerySqlNode(&yymsp[-1].minor.yy0, yymsp[0].minor.yy178, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); } - yymsp[-1].minor.yy544 = yylhsminor.yy544; + yymsp[-1].minor.yy288 = yylhsminor.yy288; break; case 166: /* sclp ::= selcollist COMMA */ -{yylhsminor.yy458 = yymsp[-1].minor.yy458;} - yymsp[-1].minor.yy458 = yylhsminor.yy458; +{yylhsminor.yy178 = yymsp[-1].minor.yy178;} + yymsp[-1].minor.yy178 = yylhsminor.yy178; break; case 167: /* sclp ::= */ -{yymsp[1].minor.yy458 = 0;} +{yymsp[1].minor.yy178 = 0;} break; case 168: /* selcollist ::= sclp distinct expr as */ { - yylhsminor.yy458 = tSqlExprListAppend(yymsp[-3].minor.yy458, yymsp[-1].minor.yy70, yymsp[-2].minor.yy0.n? &yymsp[-2].minor.yy0:0, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); + yylhsminor.yy178 = tSqlExprListAppend(yymsp[-3].minor.yy178, yymsp[-1].minor.yy134, yymsp[-2].minor.yy0.n? &yymsp[-2].minor.yy0:0, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); } - yymsp[-3].minor.yy458 = yylhsminor.yy458; + yymsp[-3].minor.yy178 = yylhsminor.yy178; break; case 169: /* selcollist ::= sclp STAR */ { tSQLExpr *pNode = tSqlExprIdValueCreate(NULL, TK_ALL); - yylhsminor.yy458 = tSqlExprListAppend(yymsp[-1].minor.yy458, pNode, 0, 0); + yylhsminor.yy178 = tSqlExprListAppend(yymsp[-1].minor.yy178, pNode, 0, 0); } - yymsp[-1].minor.yy458 = yylhsminor.yy458; + yymsp[-1].minor.yy178 = yylhsminor.yy178; break; case 170: /* as ::= AS ids */ { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } @@ -3014,320 +2972,330 @@ static YYACTIONTYPE yy_reduce( yymsp[0].minor.yy0 = yylhsminor.yy0; break; case 175: /* from ::= FROM tablelist */ -{yymsp[-1].minor.yy161 = yymsp[0].minor.yy161;} +{yymsp[-1].minor.yy193 = yymsp[0].minor.yy193;} break; case 176: /* tablelist ::= ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yylhsminor.yy161 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); - yylhsminor.yy161 = tVariantListAppendToken(yylhsminor.yy161, &yymsp[-1].minor.yy0, -1); // table alias name + yylhsminor.yy193 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); + yylhsminor.yy193 = tVariantListAppendToken(yylhsminor.yy193, &yymsp[-1].minor.yy0, -1); // table alias name } - yymsp[-1].minor.yy161 = yylhsminor.yy161; + yymsp[-1].minor.yy193 = yylhsminor.yy193; break; case 177: /* tablelist ::= ids cpxName ids */ { toTSDBType(yymsp[-2].minor.yy0.type); toTSDBType(yymsp[0].minor.yy0.type); yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - yylhsminor.yy161 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); - yylhsminor.yy161 = tVariantListAppendToken(yylhsminor.yy161, &yymsp[0].minor.yy0, -1); + yylhsminor.yy193 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); + yylhsminor.yy193 = tVariantListAppendToken(yylhsminor.yy193, &yymsp[0].minor.yy0, -1); } - yymsp[-2].minor.yy161 = yylhsminor.yy161; + yymsp[-2].minor.yy193 = yylhsminor.yy193; break; case 178: /* tablelist ::= tablelist COMMA ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yylhsminor.yy161 = tVariantListAppendToken(yymsp[-3].minor.yy161, &yymsp[-1].minor.yy0, -1); - yylhsminor.yy161 = tVariantListAppendToken(yylhsminor.yy161, &yymsp[-1].minor.yy0, -1); + yylhsminor.yy193 = tVariantListAppendToken(yymsp[-3].minor.yy193, &yymsp[-1].minor.yy0, -1); + yylhsminor.yy193 = tVariantListAppendToken(yylhsminor.yy193, &yymsp[-1].minor.yy0, -1); } - yymsp[-3].minor.yy161 = yylhsminor.yy161; + yymsp[-3].minor.yy193 = yylhsminor.yy193; break; case 179: /* tablelist ::= tablelist COMMA ids cpxName ids */ { toTSDBType(yymsp[-2].minor.yy0.type); toTSDBType(yymsp[0].minor.yy0.type); yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - yylhsminor.yy161 = tVariantListAppendToken(yymsp[-4].minor.yy161, &yymsp[-2].minor.yy0, -1); - yylhsminor.yy161 = tVariantListAppendToken(yylhsminor.yy161, &yymsp[0].minor.yy0, -1); + yylhsminor.yy193 = tVariantListAppendToken(yymsp[-4].minor.yy193, &yymsp[-2].minor.yy0, -1); + yylhsminor.yy193 = tVariantListAppendToken(yylhsminor.yy193, &yymsp[0].minor.yy0, -1); } - yymsp[-4].minor.yy161 = yylhsminor.yy161; + yymsp[-4].minor.yy193 = yylhsminor.yy193; break; case 180: /* tmvar ::= VARIABLE */ {yylhsminor.yy0 = yymsp[0].minor.yy0;} yymsp[0].minor.yy0 = yylhsminor.yy0; break; case 181: /* interval_opt ::= INTERVAL LP tmvar RP */ -{yymsp[-3].minor.yy300.interval = yymsp[-1].minor.yy0; yymsp[-3].minor.yy300.offset.n = 0; yymsp[-3].minor.yy300.offset.z = NULL; yymsp[-3].minor.yy300.offset.type = 0;} +{yymsp[-3].minor.yy392.interval = yymsp[-1].minor.yy0; yymsp[-3].minor.yy392.offset.n = 0;} break; case 182: /* interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ -{yymsp[-5].minor.yy300.interval = yymsp[-3].minor.yy0; yymsp[-5].minor.yy300.offset = yymsp[-1].minor.yy0;} +{yymsp[-5].minor.yy392.interval = yymsp[-3].minor.yy0; yymsp[-5].minor.yy392.offset = yymsp[-1].minor.yy0;} break; case 183: /* interval_opt ::= */ -{memset(&yymsp[1].minor.yy300, 0, sizeof(yymsp[1].minor.yy300));} +{memset(&yymsp[1].minor.yy392, 0, sizeof(yymsp[1].minor.yy392));} break; - case 184: /* fill_opt ::= */ -{yymsp[1].minor.yy161 = 0; } + case 184: /* session_option ::= */ +{yymsp[1].minor.yy447.col.n = 0; yymsp[1].minor.yy447.gap.n = 0;} break; - case 185: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + case 185: /* session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ +{ + yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; + yymsp[-6].minor.yy447.col = yymsp[-4].minor.yy0; + yymsp[-6].minor.yy447.gap = yymsp[-1].minor.yy0; +} + break; + case 186: /* fill_opt ::= */ +{ yymsp[1].minor.yy193 = 0; } + break; + case 187: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ { tVariant A = {0}; toTSDBType(yymsp[-3].minor.yy0.type); tVariantCreate(&A, &yymsp[-3].minor.yy0); - tVariantListInsert(yymsp[-1].minor.yy161, &A, -1, 0); - yymsp[-5].minor.yy161 = yymsp[-1].minor.yy161; + tVariantListInsert(yymsp[-1].minor.yy193, &A, -1, 0); + yymsp[-5].minor.yy193 = yymsp[-1].minor.yy193; } break; - case 186: /* fill_opt ::= FILL LP ID RP */ + case 188: /* fill_opt ::= FILL LP ID RP */ { toTSDBType(yymsp[-1].minor.yy0.type); - yymsp[-3].minor.yy161 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); + yymsp[-3].minor.yy193 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); } break; - case 187: /* sliding_opt ::= SLIDING LP tmvar RP */ + case 189: /* sliding_opt ::= SLIDING LP tmvar RP */ {yymsp[-3].minor.yy0 = yymsp[-1].minor.yy0; } break; - case 188: /* sliding_opt ::= */ + case 190: /* sliding_opt ::= */ {yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = NULL; yymsp[1].minor.yy0.type = 0; } break; - case 189: /* orderby_opt ::= */ -{yymsp[1].minor.yy161 = 0;} + case 191: /* orderby_opt ::= */ +{yymsp[1].minor.yy193 = 0;} break; - case 190: /* orderby_opt ::= ORDER BY sortlist */ -{yymsp[-2].minor.yy161 = yymsp[0].minor.yy161;} + case 192: /* orderby_opt ::= ORDER BY sortlist */ +{yymsp[-2].minor.yy193 = yymsp[0].minor.yy193;} break; - case 191: /* sortlist ::= sortlist COMMA item sortorder */ + case 193: /* sortlist ::= sortlist COMMA item sortorder */ { - yylhsminor.yy161 = tVariantListAppend(yymsp[-3].minor.yy161, &yymsp[-1].minor.yy526, yymsp[0].minor.yy452); + yylhsminor.yy193 = tVariantListAppend(yymsp[-3].minor.yy193, &yymsp[-1].minor.yy442, yymsp[0].minor.yy312); } - yymsp[-3].minor.yy161 = yylhsminor.yy161; + yymsp[-3].minor.yy193 = yylhsminor.yy193; break; - case 192: /* sortlist ::= item sortorder */ + case 194: /* sortlist ::= item sortorder */ { - yylhsminor.yy161 = tVariantListAppend(NULL, &yymsp[-1].minor.yy526, yymsp[0].minor.yy452); + yylhsminor.yy193 = tVariantListAppend(NULL, &yymsp[-1].minor.yy442, yymsp[0].minor.yy312); } - yymsp[-1].minor.yy161 = yylhsminor.yy161; + yymsp[-1].minor.yy193 = yylhsminor.yy193; break; - case 193: /* item ::= ids cpxName */ + case 195: /* item ::= ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - tVariantCreate(&yylhsminor.yy526, &yymsp[-1].minor.yy0); + tVariantCreate(&yylhsminor.yy442, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy526 = yylhsminor.yy526; + yymsp[-1].minor.yy442 = yylhsminor.yy442; break; - case 194: /* sortorder ::= ASC */ -{ yymsp[0].minor.yy452 = TSDB_ORDER_ASC; } + case 196: /* sortorder ::= ASC */ +{ yymsp[0].minor.yy312 = TSDB_ORDER_ASC; } break; - case 195: /* sortorder ::= DESC */ -{ yymsp[0].minor.yy452 = TSDB_ORDER_DESC;} + case 197: /* sortorder ::= DESC */ +{ yymsp[0].minor.yy312 = TSDB_ORDER_DESC;} break; - case 196: /* sortorder ::= */ -{ yymsp[1].minor.yy452 = TSDB_ORDER_ASC; } + case 198: /* sortorder ::= */ +{ yymsp[1].minor.yy312 = TSDB_ORDER_ASC; } break; - case 197: /* groupby_opt ::= */ -{ yymsp[1].minor.yy161 = 0;} + case 199: /* groupby_opt ::= */ +{ yymsp[1].minor.yy193 = 0;} break; - case 198: /* groupby_opt ::= GROUP BY grouplist */ -{ yymsp[-2].minor.yy161 = yymsp[0].minor.yy161;} + case 200: /* groupby_opt ::= GROUP BY grouplist */ +{ yymsp[-2].minor.yy193 = yymsp[0].minor.yy193;} break; - case 199: /* grouplist ::= grouplist COMMA item */ + case 201: /* grouplist ::= grouplist COMMA item */ { - yylhsminor.yy161 = tVariantListAppend(yymsp[-2].minor.yy161, &yymsp[0].minor.yy526, -1); + yylhsminor.yy193 = tVariantListAppend(yymsp[-2].minor.yy193, &yymsp[0].minor.yy442, -1); } - yymsp[-2].minor.yy161 = yylhsminor.yy161; + yymsp[-2].minor.yy193 = yylhsminor.yy193; break; - case 200: /* grouplist ::= item */ + case 202: /* grouplist ::= item */ { - yylhsminor.yy161 = tVariantListAppend(NULL, &yymsp[0].minor.yy526, -1); + yylhsminor.yy193 = tVariantListAppend(NULL, &yymsp[0].minor.yy442, -1); } - yymsp[0].minor.yy161 = yylhsminor.yy161; + yymsp[0].minor.yy193 = yylhsminor.yy193; break; - case 201: /* having_opt ::= */ - case 211: /* where_opt ::= */ yytestcase(yyruleno==211); - case 250: /* expritem ::= */ yytestcase(yyruleno==250); -{yymsp[1].minor.yy70 = 0;} + case 203: /* having_opt ::= */ + case 213: /* where_opt ::= */ yytestcase(yyruleno==213); + case 252: /* expritem ::= */ yytestcase(yyruleno==252); +{yymsp[1].minor.yy134 = 0;} break; - case 202: /* having_opt ::= HAVING expr */ - case 212: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==212); -{yymsp[-1].minor.yy70 = yymsp[0].minor.yy70;} + case 204: /* having_opt ::= HAVING expr */ + case 214: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==214); +{yymsp[-1].minor.yy134 = yymsp[0].minor.yy134;} break; - case 203: /* limit_opt ::= */ - case 207: /* slimit_opt ::= */ yytestcase(yyruleno==207); -{yymsp[1].minor.yy394.limit = -1; yymsp[1].minor.yy394.offset = 0;} + case 205: /* limit_opt ::= */ + case 209: /* slimit_opt ::= */ yytestcase(yyruleno==209); +{yymsp[1].minor.yy482.limit = -1; yymsp[1].minor.yy482.offset = 0;} break; - case 204: /* limit_opt ::= LIMIT signed */ - case 208: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==208); -{yymsp[-1].minor.yy394.limit = yymsp[0].minor.yy369; yymsp[-1].minor.yy394.offset = 0;} + case 206: /* limit_opt ::= LIMIT signed */ + case 210: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==210); +{yymsp[-1].minor.yy482.limit = yymsp[0].minor.yy473; yymsp[-1].minor.yy482.offset = 0;} break; - case 205: /* limit_opt ::= LIMIT signed OFFSET signed */ -{ yymsp[-3].minor.yy394.limit = yymsp[-2].minor.yy369; yymsp[-3].minor.yy394.offset = yymsp[0].minor.yy369;} + case 207: /* limit_opt ::= LIMIT signed OFFSET signed */ +{ yymsp[-3].minor.yy482.limit = yymsp[-2].minor.yy473; yymsp[-3].minor.yy482.offset = yymsp[0].minor.yy473;} break; - case 206: /* limit_opt ::= LIMIT signed COMMA signed */ -{ yymsp[-3].minor.yy394.limit = yymsp[0].minor.yy369; yymsp[-3].minor.yy394.offset = yymsp[-2].minor.yy369;} + case 208: /* limit_opt ::= LIMIT signed COMMA signed */ +{ yymsp[-3].minor.yy482.limit = yymsp[0].minor.yy473; yymsp[-3].minor.yy482.offset = yymsp[-2].minor.yy473;} break; - case 209: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ -{yymsp[-3].minor.yy394.limit = yymsp[-2].minor.yy369; yymsp[-3].minor.yy394.offset = yymsp[0].minor.yy369;} + case 211: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ +{yymsp[-3].minor.yy482.limit = yymsp[-2].minor.yy473; yymsp[-3].minor.yy482.offset = yymsp[0].minor.yy473;} break; - case 210: /* slimit_opt ::= SLIMIT signed COMMA signed */ -{yymsp[-3].minor.yy394.limit = yymsp[0].minor.yy369; yymsp[-3].minor.yy394.offset = yymsp[-2].minor.yy369;} + case 212: /* slimit_opt ::= SLIMIT signed COMMA signed */ +{yymsp[-3].minor.yy482.limit = yymsp[0].minor.yy473; yymsp[-3].minor.yy482.offset = yymsp[-2].minor.yy473;} break; - case 213: /* expr ::= LP expr RP */ -{yylhsminor.yy70 = yymsp[-1].minor.yy70; yylhsminor.yy70->token.z = yymsp[-2].minor.yy0.z; yylhsminor.yy70->token.n = (yymsp[0].minor.yy0.z - yymsp[-2].minor.yy0.z + 1);} - yymsp[-2].minor.yy70 = yylhsminor.yy70; + case 215: /* expr ::= LP expr RP */ +{yylhsminor.yy134 = yymsp[-1].minor.yy134; yylhsminor.yy134->token.z = yymsp[-2].minor.yy0.z; yylhsminor.yy134->token.n = (yymsp[0].minor.yy0.z - yymsp[-2].minor.yy0.z + 1);} + yymsp[-2].minor.yy134 = yylhsminor.yy134; break; - case 214: /* expr ::= ID */ -{ yylhsminor.yy70 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);} - yymsp[0].minor.yy70 = yylhsminor.yy70; + case 216: /* expr ::= ID */ +{ yylhsminor.yy134 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);} + yymsp[0].minor.yy134 = yylhsminor.yy134; break; - case 215: /* expr ::= ID DOT ID */ -{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy70 = tSqlExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);} - yymsp[-2].minor.yy70 = yylhsminor.yy70; + case 217: /* expr ::= ID DOT ID */ +{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy134 = tSqlExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);} + yymsp[-2].minor.yy134 = yylhsminor.yy134; break; - case 216: /* expr ::= ID DOT STAR */ -{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy70 = tSqlExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);} - yymsp[-2].minor.yy70 = yylhsminor.yy70; + case 218: /* expr ::= ID DOT STAR */ +{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy134 = tSqlExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);} + yymsp[-2].minor.yy134 = yylhsminor.yy134; break; - case 217: /* expr ::= INTEGER */ -{ yylhsminor.yy70 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);} - yymsp[0].minor.yy70 = yylhsminor.yy70; + case 219: /* expr ::= INTEGER */ +{ yylhsminor.yy134 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);} + yymsp[0].minor.yy134 = yylhsminor.yy134; break; - case 218: /* expr ::= MINUS INTEGER */ - case 219: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==219); -{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy70 = tSqlExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);} - yymsp[-1].minor.yy70 = yylhsminor.yy70; + case 220: /* expr ::= MINUS INTEGER */ + case 221: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==221); +{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy134 = tSqlExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);} + yymsp[-1].minor.yy134 = yylhsminor.yy134; break; - case 220: /* expr ::= FLOAT */ -{ yylhsminor.yy70 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);} - yymsp[0].minor.yy70 = yylhsminor.yy70; + case 222: /* expr ::= FLOAT */ +{ yylhsminor.yy134 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);} + yymsp[0].minor.yy134 = yylhsminor.yy134; break; - case 221: /* expr ::= MINUS FLOAT */ - case 222: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==222); -{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy70 = tSqlExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);} - yymsp[-1].minor.yy70 = yylhsminor.yy70; + case 223: /* expr ::= MINUS FLOAT */ + case 224: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==224); +{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy134 = tSqlExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);} + yymsp[-1].minor.yy134 = yylhsminor.yy134; break; - case 223: /* expr ::= STRING */ -{ yylhsminor.yy70 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);} - yymsp[0].minor.yy70 = yylhsminor.yy70; + case 225: /* expr ::= STRING */ +{ yylhsminor.yy134 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);} + yymsp[0].minor.yy134 = yylhsminor.yy134; break; - case 224: /* expr ::= NOW */ -{ yylhsminor.yy70 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); } - yymsp[0].minor.yy70 = yylhsminor.yy70; + case 226: /* expr ::= NOW */ +{ yylhsminor.yy134 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); } + yymsp[0].minor.yy134 = yylhsminor.yy134; break; - case 225: /* expr ::= VARIABLE */ -{ yylhsminor.yy70 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);} - yymsp[0].minor.yy70 = yylhsminor.yy70; + case 227: /* expr ::= VARIABLE */ +{ yylhsminor.yy134 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);} + yymsp[0].minor.yy134 = yylhsminor.yy134; break; - case 226: /* expr ::= BOOL */ -{ yylhsminor.yy70 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);} - yymsp[0].minor.yy70 = yylhsminor.yy70; + case 228: /* expr ::= BOOL */ +{ yylhsminor.yy134 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);} + yymsp[0].minor.yy134 = yylhsminor.yy134; break; - case 227: /* expr ::= ID LP exprlist RP */ -{ yylhsminor.yy70 = tSqlExprCreateFunction(yymsp[-1].minor.yy458, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } - yymsp[-3].minor.yy70 = yylhsminor.yy70; + case 229: /* expr ::= ID LP exprlist RP */ +{ yylhsminor.yy134 = tSqlExprCreateFunction(yymsp[-1].minor.yy178, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } + yymsp[-3].minor.yy134 = yylhsminor.yy134; break; - case 228: /* expr ::= ID LP STAR RP */ -{ yylhsminor.yy70 = tSqlExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } - yymsp[-3].minor.yy70 = yylhsminor.yy70; + case 230: /* expr ::= ID LP STAR RP */ +{ yylhsminor.yy134 = tSqlExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } + yymsp[-3].minor.yy134 = yylhsminor.yy134; break; - case 229: /* expr ::= expr IS NULL */ -{yylhsminor.yy70 = tSqlExprCreate(yymsp[-2].minor.yy70, NULL, TK_ISNULL);} - yymsp[-2].minor.yy70 = yylhsminor.yy70; + case 231: /* expr ::= expr IS NULL */ +{yylhsminor.yy134 = tSqlExprCreate(yymsp[-2].minor.yy134, NULL, TK_ISNULL);} + yymsp[-2].minor.yy134 = yylhsminor.yy134; break; - case 230: /* expr ::= expr IS NOT NULL */ -{yylhsminor.yy70 = tSqlExprCreate(yymsp[-3].minor.yy70, NULL, TK_NOTNULL);} - yymsp[-3].minor.yy70 = yylhsminor.yy70; + case 232: /* expr ::= expr IS NOT NULL */ +{yylhsminor.yy134 = tSqlExprCreate(yymsp[-3].minor.yy134, NULL, TK_NOTNULL);} + yymsp[-3].minor.yy134 = yylhsminor.yy134; break; - case 231: /* expr ::= expr LT expr */ -{yylhsminor.yy70 = tSqlExprCreate(yymsp[-2].minor.yy70, yymsp[0].minor.yy70, TK_LT);} - yymsp[-2].minor.yy70 = yylhsminor.yy70; + case 233: /* expr ::= expr LT expr */ +{yylhsminor.yy134 = tSqlExprCreate(yymsp[-2].minor.yy134, yymsp[0].minor.yy134, TK_LT);} + yymsp[-2].minor.yy134 = yylhsminor.yy134; break; - case 232: /* expr ::= expr GT expr */ -{yylhsminor.yy70 = tSqlExprCreate(yymsp[-2].minor.yy70, yymsp[0].minor.yy70, TK_GT);} - yymsp[-2].minor.yy70 = yylhsminor.yy70; + case 234: /* expr ::= expr GT expr */ +{yylhsminor.yy134 = tSqlExprCreate(yymsp[-2].minor.yy134, yymsp[0].minor.yy134, TK_GT);} + yymsp[-2].minor.yy134 = yylhsminor.yy134; break; - case 233: /* expr ::= expr LE expr */ -{yylhsminor.yy70 = tSqlExprCreate(yymsp[-2].minor.yy70, yymsp[0].minor.yy70, TK_LE);} - yymsp[-2].minor.yy70 = yylhsminor.yy70; + case 235: /* expr ::= expr LE expr */ +{yylhsminor.yy134 = tSqlExprCreate(yymsp[-2].minor.yy134, yymsp[0].minor.yy134, TK_LE);} + yymsp[-2].minor.yy134 = yylhsminor.yy134; break; - case 234: /* expr ::= expr GE expr */ -{yylhsminor.yy70 = tSqlExprCreate(yymsp[-2].minor.yy70, yymsp[0].minor.yy70, TK_GE);} - yymsp[-2].minor.yy70 = yylhsminor.yy70; + case 236: /* expr ::= expr GE expr */ +{yylhsminor.yy134 = tSqlExprCreate(yymsp[-2].minor.yy134, yymsp[0].minor.yy134, TK_GE);} + yymsp[-2].minor.yy134 = yylhsminor.yy134; break; - case 235: /* expr ::= expr NE expr */ -{yylhsminor.yy70 = tSqlExprCreate(yymsp[-2].minor.yy70, yymsp[0].minor.yy70, TK_NE);} - yymsp[-2].minor.yy70 = yylhsminor.yy70; + case 237: /* expr ::= expr NE expr */ +{yylhsminor.yy134 = tSqlExprCreate(yymsp[-2].minor.yy134, yymsp[0].minor.yy134, TK_NE);} + yymsp[-2].minor.yy134 = yylhsminor.yy134; break; - case 236: /* expr ::= expr EQ expr */ -{yylhsminor.yy70 = tSqlExprCreate(yymsp[-2].minor.yy70, yymsp[0].minor.yy70, TK_EQ);} - yymsp[-2].minor.yy70 = yylhsminor.yy70; + case 238: /* expr ::= expr EQ expr */ +{yylhsminor.yy134 = tSqlExprCreate(yymsp[-2].minor.yy134, yymsp[0].minor.yy134, TK_EQ);} + yymsp[-2].minor.yy134 = yylhsminor.yy134; break; - case 237: /* expr ::= expr BETWEEN expr AND expr */ -{ tSQLExpr* X2 = tSqlExprClone(yymsp[-4].minor.yy70); yylhsminor.yy70 = tSqlExprCreate(tSqlExprCreate(yymsp[-4].minor.yy70, yymsp[-2].minor.yy70, TK_GE), tSqlExprCreate(X2, yymsp[0].minor.yy70, TK_LE), TK_AND);} - yymsp[-4].minor.yy70 = yylhsminor.yy70; + case 239: /* expr ::= expr BETWEEN expr AND expr */ +{ tSQLExpr* X2 = tSqlExprClone(yymsp[-4].minor.yy134); yylhsminor.yy134 = tSqlExprCreate(tSqlExprCreate(yymsp[-4].minor.yy134, yymsp[-2].minor.yy134, TK_GE), tSqlExprCreate(X2, yymsp[0].minor.yy134, TK_LE), TK_AND);} + yymsp[-4].minor.yy134 = yylhsminor.yy134; break; - case 238: /* expr ::= expr AND expr */ -{yylhsminor.yy70 = tSqlExprCreate(yymsp[-2].minor.yy70, yymsp[0].minor.yy70, TK_AND);} - yymsp[-2].minor.yy70 = yylhsminor.yy70; + case 240: /* expr ::= expr AND expr */ +{yylhsminor.yy134 = tSqlExprCreate(yymsp[-2].minor.yy134, yymsp[0].minor.yy134, TK_AND);} + yymsp[-2].minor.yy134 = yylhsminor.yy134; break; - case 239: /* expr ::= expr OR expr */ -{yylhsminor.yy70 = tSqlExprCreate(yymsp[-2].minor.yy70, yymsp[0].minor.yy70, TK_OR); } - yymsp[-2].minor.yy70 = yylhsminor.yy70; + case 241: /* expr ::= expr OR expr */ +{yylhsminor.yy134 = tSqlExprCreate(yymsp[-2].minor.yy134, yymsp[0].minor.yy134, TK_OR); } + yymsp[-2].minor.yy134 = yylhsminor.yy134; break; - case 240: /* expr ::= expr PLUS expr */ -{yylhsminor.yy70 = tSqlExprCreate(yymsp[-2].minor.yy70, yymsp[0].minor.yy70, TK_PLUS); } - yymsp[-2].minor.yy70 = yylhsminor.yy70; + case 242: /* expr ::= expr PLUS expr */ +{yylhsminor.yy134 = tSqlExprCreate(yymsp[-2].minor.yy134, yymsp[0].minor.yy134, TK_PLUS); } + yymsp[-2].minor.yy134 = yylhsminor.yy134; break; - case 241: /* expr ::= expr MINUS expr */ -{yylhsminor.yy70 = tSqlExprCreate(yymsp[-2].minor.yy70, yymsp[0].minor.yy70, TK_MINUS); } - yymsp[-2].minor.yy70 = yylhsminor.yy70; + case 243: /* expr ::= expr MINUS expr */ +{yylhsminor.yy134 = tSqlExprCreate(yymsp[-2].minor.yy134, yymsp[0].minor.yy134, TK_MINUS); } + yymsp[-2].minor.yy134 = yylhsminor.yy134; break; - case 242: /* expr ::= expr STAR expr */ -{yylhsminor.yy70 = tSqlExprCreate(yymsp[-2].minor.yy70, yymsp[0].minor.yy70, TK_STAR); } - yymsp[-2].minor.yy70 = yylhsminor.yy70; + case 244: /* expr ::= expr STAR expr */ +{yylhsminor.yy134 = tSqlExprCreate(yymsp[-2].minor.yy134, yymsp[0].minor.yy134, TK_STAR); } + yymsp[-2].minor.yy134 = yylhsminor.yy134; break; - case 243: /* expr ::= expr SLASH expr */ -{yylhsminor.yy70 = tSqlExprCreate(yymsp[-2].minor.yy70, yymsp[0].minor.yy70, TK_DIVIDE);} - yymsp[-2].minor.yy70 = yylhsminor.yy70; + case 245: /* expr ::= expr SLASH expr */ +{yylhsminor.yy134 = tSqlExprCreate(yymsp[-2].minor.yy134, yymsp[0].minor.yy134, TK_DIVIDE);} + yymsp[-2].minor.yy134 = yylhsminor.yy134; break; - case 244: /* expr ::= expr REM expr */ -{yylhsminor.yy70 = tSqlExprCreate(yymsp[-2].minor.yy70, yymsp[0].minor.yy70, TK_REM); } - yymsp[-2].minor.yy70 = yylhsminor.yy70; + case 246: /* expr ::= expr REM expr */ +{yylhsminor.yy134 = tSqlExprCreate(yymsp[-2].minor.yy134, yymsp[0].minor.yy134, TK_REM); } + yymsp[-2].minor.yy134 = yylhsminor.yy134; break; - case 245: /* expr ::= expr LIKE expr */ -{yylhsminor.yy70 = tSqlExprCreate(yymsp[-2].minor.yy70, yymsp[0].minor.yy70, TK_LIKE); } - yymsp[-2].minor.yy70 = yylhsminor.yy70; + case 247: /* expr ::= expr LIKE expr */ +{yylhsminor.yy134 = tSqlExprCreate(yymsp[-2].minor.yy134, yymsp[0].minor.yy134, TK_LIKE); } + yymsp[-2].minor.yy134 = yylhsminor.yy134; break; - case 246: /* expr ::= expr IN LP exprlist RP */ -{yylhsminor.yy70 = tSqlExprCreate(yymsp[-4].minor.yy70, (tSQLExpr*)yymsp[-1].minor.yy458, TK_IN); } - yymsp[-4].minor.yy70 = yylhsminor.yy70; + case 248: /* expr ::= expr IN LP exprlist RP */ +{yylhsminor.yy134 = tSqlExprCreate(yymsp[-4].minor.yy134, (tSQLExpr*)yymsp[-1].minor.yy178, TK_IN); } + yymsp[-4].minor.yy134 = yylhsminor.yy134; break; - case 247: /* exprlist ::= exprlist COMMA expritem */ -{yylhsminor.yy458 = tSqlExprListAppend(yymsp[-2].minor.yy458,yymsp[0].minor.yy70,0, 0);} - yymsp[-2].minor.yy458 = yylhsminor.yy458; + case 249: /* exprlist ::= exprlist COMMA expritem */ +{yylhsminor.yy178 = tSqlExprListAppend(yymsp[-2].minor.yy178,yymsp[0].minor.yy134,0, 0);} + yymsp[-2].minor.yy178 = yylhsminor.yy178; break; - case 248: /* exprlist ::= expritem */ -{yylhsminor.yy458 = tSqlExprListAppend(0,yymsp[0].minor.yy70,0, 0);} - yymsp[0].minor.yy458 = yylhsminor.yy458; + case 250: /* exprlist ::= expritem */ +{yylhsminor.yy178 = tSqlExprListAppend(0,yymsp[0].minor.yy134,0, 0);} + yymsp[0].minor.yy178 = yylhsminor.yy178; break; - case 249: /* expritem ::= expr */ -{yylhsminor.yy70 = yymsp[0].minor.yy70;} - yymsp[0].minor.yy70 = yylhsminor.yy70; + case 251: /* expritem ::= expr */ +{yylhsminor.yy134 = yymsp[0].minor.yy134;} + yymsp[0].minor.yy134 = yylhsminor.yy134; break; - case 251: /* cmd ::= RESET QUERY CACHE */ + case 253: /* cmd ::= RESET QUERY CACHE */ { setDCLSQLElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} break; - case 252: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + case 254: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy161, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, -1); + SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy193, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 253: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + case 255: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -3338,14 +3306,14 @@ static YYACTIONTYPE yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 254: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + case 256: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy161, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, -1); + SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy193, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 255: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + case 257: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -3356,7 +3324,7 @@ static YYACTIONTYPE yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 256: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + case 258: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; @@ -3370,26 +3338,26 @@ static YYACTIONTYPE yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 257: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + case 259: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ { yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n; toTSDBType(yymsp[-2].minor.yy0.type); SArray* A = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); - A = tVariantListAppend(A, &yymsp[0].minor.yy526, -1); + A = tVariantListAppend(A, &yymsp[0].minor.yy442, -1); SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&yymsp[-6].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 258: /* cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ + case 260: /* cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy161, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, TSDB_SUPER_TABLE); + SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy193, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 259: /* cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ + case 261: /* cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -3400,14 +3368,14 @@ static YYACTIONTYPE yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 260: /* cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ + case 262: /* cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy161, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, TSDB_SUPER_TABLE); + SAlterTableInfo* pAlterTable = tAlterTableSqlElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy193, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 261: /* cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ + case 263: /* cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -3418,7 +3386,7 @@ static YYACTIONTYPE yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 262: /* cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ + case 264: /* cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; @@ -3432,13 +3400,13 @@ static YYACTIONTYPE yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 263: /* cmd ::= KILL CONNECTION INTEGER */ + case 265: /* cmd ::= KILL CONNECTION INTEGER */ {setKillSql(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);} break; - case 264: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ + case 266: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSql(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);} break; - case 265: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ + case 267: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSql(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);} break; default: diff --git a/src/query/tests/resultBufferTest.cpp b/src/query/tests/resultBufferTest.cpp index 7b946d858913b14dfdbb71eb1b8a8c469e6d930c..19e011774241dffa156a0ecb003041aba8253ba5 100644 --- a/src/query/tests/resultBufferTest.cpp +++ b/src/query/tests/resultBufferTest.cpp @@ -10,7 +10,7 @@ namespace { // simple test void simpleTest() { SDiskbasedResultBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 64, 1024, 4096, NULL); + int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4096, NULL); int32_t pageId = 0; int32_t groupId = 0; @@ -52,7 +52,7 @@ void simpleTest() { void writeDownTest() { SDiskbasedResultBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 64, 1024, 4*1024, NULL); + int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4*1024, NULL); int32_t pageId = 0; int32_t writePageId = 0; @@ -99,7 +99,7 @@ void writeDownTest() { void recyclePageTest() { SDiskbasedResultBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 64, 1024, 4*1024, NULL); + int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4*1024, NULL); int32_t pageId = 0; int32_t writePageId = 0; diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index a4cc316725d60510441c4badb77ce9a2805c08ca..a1ed0796a6e628a55ebe81fd87396ada990b7e73 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -15,7 +15,13 @@ #include "tsdbint.h" #define TSDB_MAX_SUBBLOCKS 8 -#define TSDB_KEY_FID(key, days, precision) ((key) / tsMsPerDay[(precision)] / (days)) +static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) { + if (key < 0) { + return (int)(-((-key) / tsMsPerDay[precision] / days + 1)); + } else { + return (int)((key / tsMsPerDay[precision] / days)); + } +} typedef struct { SRtn rtn; // retention snapshot diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 5e2e0fce1d45dc8ffceb8c92a2475df5b4da0ad3..cdde2bc91f430273a65aa3ae2ba09bc42771a024 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -20,7 +20,7 @@ static int tsdbCompareSchemaVersion(const void *key1, const void *key2); static char * getTagIndexKey(const void *pData); static STable *tsdbNewTable(); -static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper); +static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper, STable *pSTable); static void tsdbFreeTable(STable *pTable); static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx, bool lock); static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFromIdx, bool lock); @@ -43,6 +43,7 @@ static void * tsdbInsertTableAct(STsdbRepo *pRepo, int8_t act, void *buf, STabl static int tsdbRemoveTableFromStore(STsdbRepo *pRepo, STable *pTable); static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable); static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid); +static int tsdbCheckTableTagVal(SKVRow *pKVRow, STSchema *pSchema); // ------------------ OUTER FUNCTIONS ------------------ int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg) { @@ -87,7 +88,7 @@ int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg) { super = tsdbGetTableByUid(pMeta, pCfg->superUid); if (super == NULL) { // super table not exists, try to create it newSuper = true; - super = tsdbCreateTableFromCfg(pCfg, true); + super = tsdbCreateTableFromCfg(pCfg, true, NULL); if (super == NULL) goto _err; } else { if (TABLE_TYPE(super) != TSDB_SUPER_TABLE || TABLE_UID(super) != pCfg->superUid) { @@ -108,7 +109,7 @@ int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg) { } } - table = tsdbCreateTableFromCfg(pCfg, false); + table = tsdbCreateTableFromCfg(pCfg, false, super); if (table == NULL) goto _err; // Register to meta @@ -674,7 +675,7 @@ static STable *tsdbNewTable() { return pTable; } -static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper) { +static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper, STable *pSTable) { STable *pTable = NULL; size_t tsize = 0; @@ -726,6 +727,9 @@ static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper) { if (pCfg->type == TSDB_CHILD_TABLE) { TABLE_SUID(pTable) = pCfg->superUid; + if (tsdbCheckTableTagVal(pCfg->tagValues, pSTable->tagSchema) < 0) { + goto _err; + } pTable->tagVal = tdKVRowDup(pCfg->tagValues); if (pTable->tagVal == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -1302,3 +1306,20 @@ static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid) { return 0; } + +static int tsdbCheckTableTagVal(SKVRow *pKVRow, STSchema *pSchema) { + for (size_t i = 0; i < kvRowNCols(pKVRow); i++) { + SColIdx * pColIdx = kvRowColIdxAt(pKVRow, i); + STColumn *pCol = tdGetColOfID(pSchema, pColIdx->colId); + + if ((pCol == NULL) || (!IS_VAR_DATA_TYPE(pCol->type))) continue; + + void *pValue = tdGetKVRowValOfCol(pKVRow, pCol->colId); + if (varDataTLen(pValue) > pCol->bytes) { + terrno = TSDB_CODE_TDB_IVLD_TAG_VAL; + return -1; + } + } + + return 0; +} \ No newline at end of file diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index e0648c33a050fcdb5a6cd83517283092b1061d5e..0bf533d027db284a12ae155dc494fd8f9f35a5f1 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -34,11 +34,9 @@ .tid = (_checkInfo)->tableId.tid, \ .uid = (_checkInfo)->tableId.uid}) - enum { TSDB_QUERY_TYPE_ALL = 1, TSDB_QUERY_TYPE_LAST = 2, - TSDB_QUERY_TYPE_EXTERNAL = 3, }; typedef struct SQueryFilePos { @@ -68,7 +66,7 @@ typedef struct STableCheckInfo { STableId tableId; TSKEY lastKey; STable* pTableObj; - SBlockInfo* pCompInfo; + SBlockInfo* pCompInfo; int32_t compSize; int32_t numOfBlocks:29; // number of qualified data blocks not the original blocks int8_t chosen:2; // indicate which iterator should move forward @@ -78,8 +76,8 @@ typedef struct STableCheckInfo { } STableCheckInfo; typedef struct STableBlockInfo { - SBlock* compBlock; - STableCheckInfo* pTableCheckInfo; + SBlock *compBlock; + STableCheckInfo *pTableCheckInfo; } STableBlockInfo; typedef struct SBlockOrderSupporter { @@ -111,6 +109,8 @@ typedef struct STsdbQueryHandle { bool checkFiles; // check file stage bool cachelastrow; // check if last row cached bool loadExternalRow; // load time window external data rows + bool currentLoadExternalRows; // current load external rows + int32_t loadType; // block load type void* qinfo; // query info handle, for debug purpose int32_t type; // query type: retrieve all data blocks, 2. retrieve only last row, 3. retrieve direct prev|next rows SDFileSet* pFileGroup; @@ -147,6 +147,8 @@ static int32_t tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2); static int32_t doGetExternalRow(STsdbQueryHandle* pQueryHandle, int16_t type, SMemRef* pMemRef); static void* doFreeColumnInfoData(SArray* pColumnInfoData); +static void* destroyTableCheckInfo(SArray* pTableCheckInfo); +static bool tsdbGetExternalRow(TsdbQueryHandleT pHandle); static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { pBlockLoadInfo->slot = -1; @@ -294,40 +296,48 @@ static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STa taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar); size_t gsize = taosArrayGetSize(pTableCheckInfo); - + for (int32_t i = 0; i < gsize; ++i) { STableCheckInfo* pInfo = (STableCheckInfo*) taosArrayGet(pTableCheckInfo, i); - taosArrayPush(pTable, &pInfo->pTableObj); } *psTable = pTable; - return pTableCheckInfo; } -static SArray* createCheckInfoFromCheckInfo(SArray* pTableCheckInfo, TSKEY skey, SArray** psTable) { - size_t si = taosArrayGetSize(pTableCheckInfo); - SArray* pNew = taosArrayInit(si, sizeof(STableCheckInfo)); - if (pNew == NULL) { - return NULL; +static void resetCheckInfo(STsdbQueryHandle* pQueryHandle) { + size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + assert(numOfTables >= 1); + + // todo apply the lastkey of table check to avoid to load header file + for (int32_t i = 0; i < numOfTables; ++i) { + STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i); + pCheckInfo->lastKey = pQueryHandle->window.skey; + pCheckInfo->iter = tSkipListDestroyIter(pCheckInfo->iter); + pCheckInfo->iiter = tSkipListDestroyIter(pCheckInfo->iiter); + pCheckInfo->initBuf = false; + + if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + assert(pCheckInfo->lastKey >= pQueryHandle->window.skey); + } else { + assert(pCheckInfo->lastKey <= pQueryHandle->window.skey); + } } +} - SArray* pTable = taosArrayInit(si, sizeof(STable*)); +// only one table, not need to sort again +static SArray* createCheckInfoFromCheckInfo(STableCheckInfo* pCheckInfo, TSKEY skey, SArray** psTable) { + SArray* pNew = taosArrayInit(1, sizeof(STableCheckInfo)); + SArray* pTable = taosArrayInit(1, sizeof(STable*)); - for (int32_t j = 0; j < si; ++j) { - STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pTableCheckInfo, j); - STableCheckInfo info = { .lastKey = skey, .pTableObj = pCheckInfo->pTableObj}; + STableCheckInfo info = { .lastKey = skey, .pTableObj = pCheckInfo->pTableObj}; - info.tableId = pCheckInfo->tableId; - taosArrayPush(pNew, &info); - taosArrayPush(pTable, &pCheckInfo->pTableObj); - } + info.tableId = pCheckInfo->tableId; + taosArrayPush(pNew, &info); + taosArrayPush(pTable, &pCheckInfo->pTableObj); *psTable = pTable; - - // it is ordered already, no need to sort again. - taosArraySort(pNew, tsdbCheckInfoCompar); return pNew; } @@ -351,14 +361,15 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pC pQueryHandle->locateStart = false; pQueryHandle->pMemRef = pMemRef; pQueryHandle->loadExternalRow = pCond->loadExternalRows; + pQueryHandle->currentLoadExternalRows = pCond->loadExternalRows; + + pQueryHandle->loadType = pCond->type; if (tsdbInitReadH(&pQueryHandle->rhelper, (STsdbRepo*)tsdb) != 0) { goto out_of_memory; } - //tsdbMayTakeMemSnapshot(pQueryHandle); assert(pCond != NULL && pCond->numOfCols > 0 && pMemRef != NULL); - if (ASCENDING_TRAVERSE(pCond->order)) { assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey); } else { @@ -388,7 +399,9 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pC pQueryHandle->statis[i].colId = colInfo.info.colId; } - pQueryHandle->defaultLoadColumn = getDefaultLoadColumns(pQueryHandle, true); + if (pCond->numOfCols > 0) { + pQueryHandle->defaultLoadColumn = getDefaultLoadColumns(pQueryHandle, true); + } STsdbMeta* pMeta = tsdbGetMeta(tsdb); assert(pMeta != NULL); @@ -433,6 +446,74 @@ TsdbQueryHandleT* tsdbQueryTables(STsdbRepo* tsdb, STsdbQueryCond* pCond, STable return (TsdbQueryHandleT) pQueryHandle; } +void tsdbResetQueryHandle(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond) { + STsdbQueryHandle* pQueryHandle = queryHandle; + + pQueryHandle->order = pCond->order; + pQueryHandle->window = pCond->twindow; + pQueryHandle->type = TSDB_QUERY_TYPE_ALL; + pQueryHandle->cur.fid = -1; + pQueryHandle->cur.win = TSWINDOW_INITIALIZER; + pQueryHandle->checkFiles = true; + pQueryHandle->activeIndex = 0; // current active table index + pQueryHandle->locateStart = false; + pQueryHandle->loadExternalRow = pCond->loadExternalRows; + + if (ASCENDING_TRAVERSE(pCond->order)) { + assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey); + } else { + assert(pQueryHandle->window.skey >= pQueryHandle->window.ekey); + } + + // allocate buffer in order to load data blocks from file + memset(pQueryHandle->statis, 0, sizeof(SDataStatis)); + + tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); + tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); + + resetCheckInfo(pQueryHandle); +} + +void tsdbResetQueryHandleForNewTable(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond, STableGroupInfo* groupList) { + STsdbQueryHandle* pQueryHandle = queryHandle; + + pQueryHandle->order = pCond->order; + pQueryHandle->window = pCond->twindow; + pQueryHandle->type = TSDB_QUERY_TYPE_ALL; + pQueryHandle->cur.fid = -1; + pQueryHandle->cur.win = TSWINDOW_INITIALIZER; + pQueryHandle->checkFiles = true; + pQueryHandle->activeIndex = 0; // current active table index + pQueryHandle->locateStart = false; + pQueryHandle->loadExternalRow = pCond->loadExternalRows; + + if (ASCENDING_TRAVERSE(pCond->order)) { + assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey); + } else { + assert(pQueryHandle->window.skey >= pQueryHandle->window.ekey); + } + + // allocate buffer in order to load data blocks from file + memset(pQueryHandle->statis, 0, sizeof(SDataStatis)); + + tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); + tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); + + SArray* pTable = NULL; + STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb); + + pQueryHandle->pTableCheckInfo = destroyTableCheckInfo(pQueryHandle->pTableCheckInfo); + + pQueryHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pQueryHandle, groupList, pMeta, &pTable); + if (pQueryHandle->pTableCheckInfo == NULL) { + tsdbCleanupQueryHandle(pQueryHandle); + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + } + + pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); + pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); +} + TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pMemRef) { pCond->twindow = updateLastrowForEachGroup(groupList); @@ -469,12 +550,45 @@ SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle) { return res; } +// leave only one table for each group +static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGroupList) { + assert(pGroupList); + size_t numOfGroup = taosArrayGetSize(pGroupList->pGroupList); + + STableGroupInfo* pNew = calloc(1, sizeof(STableGroupInfo)); + pNew->pGroupList = taosArrayInit(numOfGroup, sizeof(SArray)); + + for(int32_t i = 0; i < numOfGroup; ++i) { + SArray* oneGroup = taosArrayGetP(pGroupList->pGroupList, i); + size_t numOfTables = taosArrayGetSize(oneGroup); + + SArray* px = taosArrayInit(4, sizeof(STableKeyInfo)); + for (int32_t j = 0; j < numOfTables; ++j) { + STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(oneGroup, j); + if (window->skey <= pInfo->lastKey && ((STable*)pInfo->pTable)->lastKey != TSKEY_INITIAL_VAL) { + taosArrayPush(px, pInfo); + pNew->numOfTables += 1; + break; + } + } + + // there are no data in this group + if (taosArrayGetSize(px) == 0) { + taosArrayDestroy(px); + } else { + taosArrayPush(pNew->pGroupList, &px); + } + } + + return pNew; +} + TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pRef) { - STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo, pRef); + STableGroupInfo* pNew = trimTableGroup(&pCond->twindow, groupList); + + STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, pNew, qinfo, pRef); pQueryHandle->loadExternalRow = true; - if (pQueryHandle != NULL) { - changeQueryHandleForInterpQuery(pQueryHandle); - } + pQueryHandle->currentLoadExternalRows = true; return pQueryHandle; } @@ -769,77 +883,94 @@ static int32_t binarySearchForBlock(SBlock* pBlock, int32_t numOfBlocks, TSKEY s return midSlot; } -static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlocks) { - // load all the comp offset value for all tables in this file - int32_t code = TSDB_CODE_SUCCESS; +static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t index, int32_t* numOfBlocks) { + int32_t code = 0; - *numOfBlocks = 0; - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, index); + pCheckInfo->numOfBlocks = 0; - for (int32_t i = 0; i < numOfTables; ++i) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); - pCheckInfo->numOfBlocks = 0; + if (tsdbSetReadTable(&pQueryHandle->rhelper, pCheckInfo->pTableObj) != TSDB_CODE_SUCCESS) { + code = terrno; + return code; + } - if (tsdbSetReadTable(&pQueryHandle->rhelper, pCheckInfo->pTableObj) != TSDB_CODE_SUCCESS) { - code = terrno; - break; - } + SBlockIdx* compIndex = pQueryHandle->rhelper.pBlkIdx; + + // no data block in this file, try next file + if (compIndex == NULL || compIndex->uid != pCheckInfo->tableId.uid) { + return 0; // no data blocks in the file belongs to pCheckInfo->pTable + } - SBlockIdx* compIndex = pQueryHandle->rhelper.pBlkIdx; + if (pCheckInfo->compSize < (int32_t)compIndex->len) { + assert(compIndex->len > 0); - // no data block in this file, try next file - if (compIndex == NULL || compIndex->uid != pCheckInfo->tableId.uid) { - continue; // no data blocks in the file belongs to pCheckInfo->pTable + char* t = realloc(pCheckInfo->pCompInfo, compIndex->len); + if (t == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + code = TSDB_CODE_TDB_OUT_OF_MEMORY; + return code; } - if (pCheckInfo->compSize < (int32_t)compIndex->len) { - assert(compIndex->len > 0); + pCheckInfo->pCompInfo = (SBlockInfo*)t; + pCheckInfo->compSize = compIndex->len; + } - char* t = realloc(pCheckInfo->pCompInfo, compIndex->len); - if (t == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - code = TSDB_CODE_TDB_OUT_OF_MEMORY; - break; - } + tsdbLoadBlockInfo(&(pQueryHandle->rhelper), (void*)(pCheckInfo->pCompInfo)); + SBlockInfo* pCompInfo = pCheckInfo->pCompInfo; - pCheckInfo->pCompInfo = (SBlockInfo*) t; - pCheckInfo->compSize = compIndex->len; - } + TSKEY s = TSKEY_INITIAL_VAL, e = TSKEY_INITIAL_VAL; - tsdbLoadBlockInfo(&(pQueryHandle->rhelper), (void *)(pCheckInfo->pCompInfo)); - SBlockInfo* pCompInfo = pCheckInfo->pCompInfo; + if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + assert(pCheckInfo->lastKey <= pQueryHandle->window.ekey && pQueryHandle->window.skey <= pQueryHandle->window.ekey); + } else { + assert(pCheckInfo->lastKey >= pQueryHandle->window.ekey && pQueryHandle->window.skey >= pQueryHandle->window.ekey); + } - TSKEY s = TSKEY_INITIAL_VAL, e = TSKEY_INITIAL_VAL; + s = MIN(pCheckInfo->lastKey, pQueryHandle->window.ekey); + e = MAX(pCheckInfo->lastKey, pQueryHandle->window.ekey); - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - assert(pCheckInfo->lastKey <= pQueryHandle->window.ekey && pQueryHandle->window.skey <= pQueryHandle->window.ekey); - } else { - assert(pCheckInfo->lastKey >= pQueryHandle->window.ekey && pQueryHandle->window.skey >= pQueryHandle->window.ekey); - } + // discard the unqualified data block based on the query time window + int32_t start = binarySearchForBlock(pCompInfo->blocks, compIndex->numOfBlocks, s, TSDB_ORDER_ASC); + int32_t end = start; - s = MIN(pCheckInfo->lastKey, pQueryHandle->window.ekey); - e = MAX(pCheckInfo->lastKey, pQueryHandle->window.ekey); + if (s > pCompInfo->blocks[start].keyLast) { + return 0; + } - // discard the unqualified data block based on the query time window - int32_t start = binarySearchForBlock(pCompInfo->blocks, compIndex->numOfBlocks, s, TSDB_ORDER_ASC); - int32_t end = start; + // todo speedup the procedure of located end block + while (end < (int32_t)compIndex->numOfBlocks && (pCompInfo->blocks[end].keyFirst <= e)) { + end += 1; + } - if (s > pCompInfo->blocks[start].keyLast) { - continue; - } + pCheckInfo->numOfBlocks = (end - start); - // todo speedup the procedure of located end block - while (end < (int32_t)compIndex->numOfBlocks && (pCompInfo->blocks[end].keyFirst <= e)) { - end += 1; - } + if (start > 0) { + memmove(pCompInfo->blocks, &pCompInfo->blocks[start], pCheckInfo->numOfBlocks * sizeof(SBlock)); + } - pCheckInfo->numOfBlocks = (end - start); + (*numOfBlocks) += pCheckInfo->numOfBlocks; + return 0; +} - if (start > 0) { - memmove(pCompInfo->blocks, &pCompInfo->blocks[start], pCheckInfo->numOfBlocks * sizeof(SBlock)); - } +static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlocks) { + // load all the comp offset value for all tables in this file + int32_t code = TSDB_CODE_SUCCESS; + *numOfBlocks = 0; + + size_t numOfTables = 0; + if (pQueryHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { + code = loadBlockInfo(pQueryHandle, pQueryHandle->activeIndex, numOfBlocks); + } else if (pQueryHandle->loadType == BLOCK_LOAD_OFFSET_SEQ_ORDER) { + numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - (*numOfBlocks) += pCheckInfo->numOfBlocks; + for (int32_t i = 0; i < numOfTables; ++i) { + code = loadBlockInfo(pQueryHandle, i, numOfBlocks); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + } else { + assert(0); } return code; @@ -1268,7 +1399,7 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, case TSDB_DATA_TYPE_DOUBLE: SET_DOUBLE_PTR(pData, value); break; - case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_TIMESTAMP: *(TSKEY *)pData = tdGetKey(*(TKEY *)value); break; default: @@ -1730,6 +1861,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO memset(pQueryHandle->pDataBlockInfo, 0, size); *numOfAllocBlocks = numOfBlocks; + // access data blocks according to the offset of each block in asc/desc order. int32_t numOfTables = (int32_t)taosArrayGetSize(pQueryHandle->pTableCheckInfo); SBlockOrderSupporter sup = {0}; @@ -1959,6 +2091,91 @@ static void moveToNextDataBlockInCurrentFile(STsdbQueryHandle* pQueryHandle) { cur->blockCompleted = false; } +int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist* pTableBlockInfo) { + STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) queryHandle; + + pTableBlockInfo->totalSize = 0; + STsdbFS* pFileHandle = REPO_FS(pQueryHandle->pTsdb); + + // find the start data block in file + pQueryHandle->locateStart = true; + STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; + int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision); + + tsdbRLockFS(pFileHandle); + tsdbFSIterInit(&pQueryHandle->fileIter, pFileHandle, pQueryHandle->order); + tsdbFSIterSeek(&pQueryHandle->fileIter, fid); + tsdbUnLockFS(pFileHandle); + + pTableBlockInfo->numOfFiles += 1; + + int32_t code = TSDB_CODE_SUCCESS; + int32_t numOfBlocks = 0; + int32_t numOfTables = (int32_t)taosArrayGetSize(pQueryHandle->pTableCheckInfo); + STimeWindow win = TSWINDOW_INITIALIZER; + + while (true) { + numOfBlocks = 0; + tsdbRLockFS(REPO_FS(pQueryHandle->pTsdb)); + + if ((pQueryHandle->pFileGroup = tsdbFSIterNext(&pQueryHandle->fileIter)) == NULL) { + tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + break; + } + + tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pQueryHandle->pFileGroup->fid, &win.skey, &win.ekey); + + // current file are not overlapped with query time window, ignore remain files + if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) || + (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) { + tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %p", pQueryHandle, + pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qinfo); + pQueryHandle->pFileGroup = NULL; + break; + } + + pTableBlockInfo->numOfFiles += 1; + if (tsdbSetAndOpenReadFSet(&pQueryHandle->rhelper, pQueryHandle->pFileGroup) < 0) { + tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + code = terrno; + break; + } + + tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + + if (tsdbLoadBlockIdx(&pQueryHandle->rhelper) < 0) { + code = terrno; + break; + } + + if ((code = getFileCompInfo(pQueryHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) { + break; + } + + tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %p", pQueryHandle, numOfBlocks, numOfTables, + pQueryHandle->pFileGroup->fid, pQueryHandle->qinfo); + + if (numOfBlocks == 0) { + continue; + } + + for (int32_t i = 0; i < numOfTables; ++i) { + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + + SBlock* pBlock = pCheckInfo->pCompInfo->blocks; + for (int32_t j = 0; j < pCheckInfo->numOfBlocks; ++j) { + pTableBlockInfo->totalSize += pBlock[j].len; + + int32_t numOfRows = pBlock[j].numOfRows; + taosArrayPush(pTableBlockInfo->dataBlockInfos, &numOfRows); + } + } + } + + return code; +} + static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists) { STsdbFS* pFileHandle = REPO_FS(pQueryHandle->pTsdb); SQueryFilePos* cur = &pQueryHandle->cur; @@ -2017,19 +2234,14 @@ static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) { pQueryHandle->activeIndex += 1; } - if (pQueryHandle->loadExternalRow && pQueryHandle->window.skey == pQueryHandle->window.ekey) { - SMemRef* pMemRef = pQueryHandle->pMemRef; - doGetExternalRow(pQueryHandle, TSDB_PREV_ROW, pMemRef); - doGetExternalRow(pQueryHandle, TSDB_NEXT_ROW, pMemRef); - } - // no data in memtable or imemtable, decrease the memory reference. - tsdbMayUnTakeMemSnapshot(pQueryHandle); + // TODO !! +// tsdbMayUnTakeMemSnapshot(pQueryHandle); return false; } //todo not unref yet, since it is not support multi-group interpolation query -static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { +static UNUSED_FUNC void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { // filter the queried time stamp in the first place STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; @@ -2152,149 +2364,164 @@ static void destroyHelper(void* param) { free(param); } -// handle data in cache situation -bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; - - int64_t stime = taosGetTimestampUs(); - int64_t elapsedTime = stime; - - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - assert(numOfTables > 0); - - if (pQueryHandle->type == TSDB_QUERY_TYPE_LAST && pQueryHandle->cachelastrow) { - // the last row is cached in buffer, return it directly. - // here note that the pQueryHandle->window must be the TS_INITIALIZER - int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); - SQueryFilePos* cur = &pQueryHandle->cur; - - SDataRow pRow = NULL; - TSKEY key = TSKEY_INITIAL_VAL; - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; - - if (++pQueryHandle->activeIndex < numOfTables) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); - int32_t ret = tsdbGetCachedLastRow(pCheckInfo->pTableObj, &pRow, &key); - if (ret != TSDB_CODE_SUCCESS) { - return false; - } - - copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, 0, pRow, numOfCols, pCheckInfo->pTableObj, NULL); - tfree(pRow); - - // update the last key value - pCheckInfo->lastKey = key + step; - - cur->rows = 1; // only one row - cur->lastKey = key + step; - cur->mixBlock = true; - cur->win.skey = key; - cur->win.ekey = key; - - return true; - } - - return false; - } - +static bool loadBlockOfActiveTable(STsdbQueryHandle* pQueryHandle) { if (pQueryHandle->checkFiles) { // check if the query range overlaps with the file data block bool exists = true; int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); if (code != TSDB_CODE_SUCCESS) { - pQueryHandle->activeIndex = 0; pQueryHandle->checkFiles = false; - return false; } if (exists) { - pQueryHandle->cost.checkForNextTime += (taosGetTimestampUs() - stime); + if (pQueryHandle->currentLoadExternalRows && pQueryHandle->window.skey == pQueryHandle->window.ekey) { + SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, 0); + assert(*(int64_t*)pColInfo->pData == pQueryHandle->window.skey); + } + + pQueryHandle->currentLoadExternalRows = false; // clear the flag, since the exact matched row is found. return exists; } - pQueryHandle->activeIndex = 0; pQueryHandle->checkFiles = false; } - // TODO: opt by consider the scan order - bool ret = doHasDataInBuffer(pQueryHandle); - terrno = TSDB_CODE_SUCCESS; + if (hasMoreDataInCache(pQueryHandle)) { + pQueryHandle->currentLoadExternalRows = false; + return true; + } - elapsedTime = taosGetTimestampUs() - stime; - pQueryHandle->cost.checkForNextTime += elapsedTime; - return ret; + // current result is empty + if (pQueryHandle->currentLoadExternalRows && pQueryHandle->window.skey == pQueryHandle->window.ekey && pQueryHandle->cur.rows == 0) { + SMemRef* pMemRef = pQueryHandle->pMemRef; + + doGetExternalRow(pQueryHandle, TSDB_PREV_ROW, pMemRef); + doGetExternalRow(pQueryHandle, TSDB_NEXT_ROW, pMemRef); + + bool result = tsdbGetExternalRow(pQueryHandle); + + pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); + pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); + pQueryHandle->currentLoadExternalRows = false; + + return result; + } + + return false; } -bool tsdbNextDataBlockWithoutMerge(TsdbQueryHandleT* pHandle) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; +static bool loadCachedLastRow(STsdbQueryHandle* pQueryHandle) { + // the last row is cached in buffer, return it directly. + // here note that the pQueryHandle->window must be the TS_INITIALIZER + int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); + size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + assert(numOfTables > 0 && numOfCols > 0); - int64_t stime = taosGetTimestampUs(); - int64_t elapsedTime = stime; + SQueryFilePos* cur = &pQueryHandle->cur; - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - assert(numOfTables > 0); + SDataRow pRow = NULL; + TSKEY key = TSKEY_INITIAL_VAL; + int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; - if (pQueryHandle->type == TSDB_QUERY_TYPE_LAST && pQueryHandle->cachelastrow) { - // the last row is cached in buffer, return it directly. - // here note that the pQueryHandle->window must be the TS_INITIALIZER - int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); - SQueryFilePos* cur = &pQueryHandle->cur; - - SDataRow pRow = NULL; - TSKEY key = TSKEY_INITIAL_VAL; - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; - - if (++pQueryHandle->activeIndex < numOfTables) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); - int32_t ret = tsdbGetCachedLastRow(pCheckInfo->pTableObj, &pRow, &key); - if (ret != TSDB_CODE_SUCCESS) { - return false; - } + if (++pQueryHandle->activeIndex < numOfTables) { + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); + int32_t ret = tsdbGetCachedLastRow(pCheckInfo->pTableObj, &pRow, &key); + if (ret != TSDB_CODE_SUCCESS) { + return false; + } - copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, 0, pRow, numOfCols, pCheckInfo->pTableObj, NULL); - tfree(pRow); + copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, 0, pRow, numOfCols, pCheckInfo->pTableObj, NULL); + tfree(pRow); - // update the last key value - pCheckInfo->lastKey = key + step; + // update the last key value + pCheckInfo->lastKey = key + step; - cur->rows = 1; // only one row - cur->lastKey = key + step; - cur->mixBlock = true; - cur->win.skey = key; - cur->win.ekey = key; + cur->rows = 1; // only one row + cur->lastKey = key + step; + cur->mixBlock = true; + cur->win.skey = key; + cur->win.ekey = key; + + return true; + } + + return false; +} + +static bool loadDataBlockFromTableSeq(STsdbQueryHandle* pQueryHandle) { + size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + assert(numOfTables > 0); + + int64_t stime = taosGetTimestampUs(); + while(pQueryHandle->activeIndex < numOfTables) { + if (loadBlockOfActiveTable(pQueryHandle)) { return true; } - return false; + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); + pCheckInfo->numOfBlocks = 0; + + pQueryHandle->activeIndex += 1; + pQueryHandle->locateStart = false; + pQueryHandle->checkFiles = true; + pQueryHandle->cur.rows = 0; + pQueryHandle->currentLoadExternalRows = pQueryHandle->loadExternalRow; + + terrno = TSDB_CODE_SUCCESS; + + int64_t elapsedTime = taosGetTimestampUs() - stime; + pQueryHandle->cost.checkForNextTime += elapsedTime; } - if (pQueryHandle->checkFiles) { - // check if the query range overlaps with the file data block - bool exists = true; + return false; +} + +// handle data in cache situation +bool tsdbNextDataBlock(TsdbQueryHandleT pHandle) { + STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; + + int64_t stime = taosGetTimestampUs(); + int64_t elapsedTime = stime; + + if (pQueryHandle->type == TSDB_QUERY_TYPE_LAST && pQueryHandle->cachelastrow) { + return loadCachedLastRow(pQueryHandle); + } + + if (pQueryHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { + return loadDataBlockFromTableSeq(pQueryHandle); + } else { // loadType == RR and Offset Order + if (pQueryHandle->checkFiles) { + // check if the query range overlaps with the file data block + bool exists = true; + + int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); + if (code != TSDB_CODE_SUCCESS) { + pQueryHandle->activeIndex = 0; + pQueryHandle->checkFiles = false; + + return false; + } + + if (exists) { + pQueryHandle->cost.checkForNextTime += (taosGetTimestampUs() - stime); + return exists; + } - int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); - if (code != TSDB_CODE_SUCCESS) { pQueryHandle->activeIndex = 0; pQueryHandle->checkFiles = false; - - return false; } - if (exists) { - pQueryHandle->cost.checkForNextTime += (taosGetTimestampUs() - stime); - return exists; - } + // TODO: opt by consider the scan order + bool ret = doHasDataInBuffer(pQueryHandle); + terrno = TSDB_CODE_SUCCESS; - pQueryHandle->activeIndex = 0; - pQueryHandle->checkFiles = false; + elapsedTime = taosGetTimestampUs() - stime; + pQueryHandle->cost.checkForNextTime += elapsedTime; + return ret; } - - elapsedTime = taosGetTimestampUs() - stime; - pQueryHandle->cost.checkForNextTime += elapsedTime; - return false; } static int32_t doGetExternalRow(STsdbQueryHandle* pQueryHandle, int16_t type, SMemRef* pMemRef) { @@ -2342,7 +2569,7 @@ static int32_t doGetExternalRow(STsdbQueryHandle* pQueryHandle, int16_t type, SM } // load the previous row - STsdbQueryCond cond = {.numOfCols = numOfCols, .loadExternalRows = false,}; + STsdbQueryCond cond = {.numOfCols = numOfCols, .loadExternalRows = false, .type = BLOCK_LOAD_OFFSET_SEQ_ORDER}; if (type == TSDB_PREV_ROW) { cond.order = TSDB_ORDER_DESC; cond.twindow = (STimeWindow){pQueryHandle->window.skey, INT64_MIN}; @@ -2363,20 +2590,20 @@ static int32_t doGetExternalRow(STsdbQueryHandle* pQueryHandle, int16_t type, SM } pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qinfo, pMemRef); - tfree(cond.colList); + // current table, only one table + STableCheckInfo* pCurrent = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); + SArray* psTable = NULL; - - pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pQueryHandle->pTableCheckInfo, pSecQueryHandle->window.skey, &psTable); + pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pCurrent, pSecQueryHandle->window.skey, &psTable); if (pSecQueryHandle->pTableCheckInfo == NULL) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; goto out_of_memory; } - - tsdbMayTakeMemSnapshot(pSecQueryHandle, psTable); + tsdbMayTakeMemSnapshot(pSecQueryHandle, psTable); if (!tsdbNextDataBlock((void*)pSecQueryHandle)) { // no result in current query, free the corresponding result rows structure if (type == TSDB_PREV_ROW) { @@ -2406,10 +2633,35 @@ out_of_memory: return terrno; } -SArray* tsdbGetExternalRow(TsdbQueryHandleT *pHandle, SMemRef* pMemRef, int16_t type) { +bool tsdbGetExternalRow(TsdbQueryHandleT pHandle) { STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; - assert(type == TSDB_PREV_ROW || type == TSDB_NEXT_ROW); - return (type == TSDB_PREV_ROW)? pQueryHandle->prev:pQueryHandle->next; + SQueryFilePos* cur = &pQueryHandle->cur; + + cur->fid = INT32_MIN; + cur->mixBlock = true; + if (pQueryHandle->prev == NULL || pQueryHandle->next == NULL) { + cur->rows = 0; + return false; + } + + int32_t numOfCols = (int32_t) QH_GET_NUM_OF_COLS(pQueryHandle); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoData* first = taosArrayGet(pQueryHandle->prev, i); + + memcpy(pColInfoData->pData, first->pData, pColInfoData->info.bytes); + + SColumnInfoData* sec = taosArrayGet(pQueryHandle->next, i); + memcpy(((char*)pColInfoData->pData) + pColInfoData->info.bytes, sec->pData, pColInfoData->info.bytes); + + if (i == 0 && pColInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP) { + cur->win.skey = *(TSKEY*)pColInfoData->pData; + cur->win.ekey = *(TSKEY*)(((char*)pColInfoData->pData) + TSDB_KEYSIZE); + } + } + + cur->rows = 2; + return true; } /* @@ -2949,10 +3201,10 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, cons goto _error; } - pGroupInfo->numOfTables = taosArrayGetSize(res); + pGroupInfo->numOfTables = (uint32_t) taosArrayGetSize(res); pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, skey); - tsdbDebug("%p no table name/tag condition, all tables belong to one group, numOfTables:%" PRIzu "", tsdb, pGroupInfo->numOfTables); + tsdbDebug("%p no table name/tag condition, all tables belong to one group, numOfTables:%u", tsdb, pGroupInfo->numOfTables); taosArrayDestroy(res); if (tsdbUnlockRepoMeta(tsdb) < 0) goto _error; @@ -2994,10 +3246,10 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, cons } END_TRY doQueryTableList(pTable, res, expr); - pGroupInfo->numOfTables = taosArrayGetSize(res); + pGroupInfo->numOfTables = (uint32_t)taosArrayGetSize(res); pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, skey); - tsdbDebug("%p stable tid:%d, uid:%"PRIu64" query, numOfTables:%" PRIzu ", belong to %" PRIzu " groups", tsdb, pTable->tableId.tid, + tsdbDebug("%p stable tid:%d, uid:%"PRIu64" query, numOfTables:%u, belong to %" PRIzu " groups", tsdb, pTable->tableId.tid, pTable->tableId.uid, pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList)); taosArrayDestroy(res); @@ -3074,7 +3326,7 @@ int32_t tsdbGetTableGroupFromIdList(STsdbRepo* tsdb, SArray* pTableIdList, STabl return terrno; } - pGroupInfo->numOfTables = taosArrayGetSize(group); + pGroupInfo->numOfTables = (uint32_t) taosArrayGetSize(group); if (pGroupInfo->numOfTables > 0) { taosArrayPush(pGroupInfo->pGroupList, &group); } else { @@ -3099,23 +3351,26 @@ static void* doFreeColumnInfoData(SArray* pColumnInfoData) { return NULL; } +static void* destroyTableCheckInfo(SArray* pTableCheckInfo) { + size_t size = taosArrayGetSize(pTableCheckInfo); + for (int32_t i = 0; i < size; ++i) { + STableCheckInfo* p = taosArrayGet(pTableCheckInfo, i); + destroyTableMemIterator(p); + + tfree(p->pCompInfo); + } + + taosArrayDestroy(pTableCheckInfo); + return NULL; +} + void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*)queryHandle; if (pQueryHandle == NULL) { return; } - - if (pQueryHandle->pTableCheckInfo != NULL) { - size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - for (int32_t i = 0; i < size; ++i) { - STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); - destroyTableMemIterator(pTableCheckInfo); - - tfree(pTableCheckInfo->pCompInfo); - } - taosArrayDestroy(pQueryHandle->pTableCheckInfo); - } + pQueryHandle->pTableCheckInfo = destroyTableCheckInfo(pQueryHandle->pTableCheckInfo); pQueryHandle->pColumns = doFreeColumnInfoData(pQueryHandle->pColumns); taosArrayDestroy(pQueryHandle->defaultLoadColumn); diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h index 63e62a54c2f679920100b7137cb49377da90789f..f2e268c2d4dfe210dfbfd9b94ee74a4f87848361 100644 --- a/src/util/inc/tarray.h +++ b/src/util/inc/tarray.h @@ -25,7 +25,8 @@ extern "C" { #define TARRAY_MIN_SIZE 8 #define TARRAY_GET_ELEM(array, index) ((void*)((char*)((array)->pData) + (index) * (array)->elemSize)) -#define TARRAY_ELEM_IDX(array, ele) (POINTER_DISTANCE(ele, (array)->pData) / (array)->elemSize) +#define TARRAY_ELEM_IDX(array, ele) (POINTER_DISTANCE(ele, (array)->pData) / (array)->elemSize) +#define TARRAY_GET_START(array) ((array)->pData) typedef struct SArray { size_t size; diff --git a/src/util/inc/tbuffer.h b/src/util/inc/tbuffer.h index e2bdb815d7ac8ecd09b62a3a62897b78ebf3c4b7..b19984b4be047108fd9848a2a7eddbac2c0b78a4 100644 --- a/src/util/inc/tbuffer.h +++ b/src/util/inc/tbuffer.h @@ -73,14 +73,14 @@ int main( int argc, char** argv ) { } */ -typedef struct { +typedef struct SBufferReader { bool endian; const char* data; size_t pos; size_t size; } SBufferReader; -typedef struct { +typedef struct SBufferWriter { bool endian; char* data; size_t pos; diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 69bcd079c0157ec19292fa88575de05092387b9b..ef68499b889d33544e601eea9ddd8cd5efe3f534 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -442,4 +442,4 @@ void vnodeWaitReadCompleted(SVnodeObj *pVnode) { vTrace("vgId:%d, queued rmsg num:%d", pVnode->vgId, pVnode->queuedRMsg); taosMsleep(10); } -} \ No newline at end of file +} diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 43506c68d51b134347df48080ef7f3223b379775..0fe94bad1d6b36aa1be5d47428dae19e918ce6e4 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1785,7 +1785,7 @@ class TdSuperTable: 'top(speed, 50)', # TODO: not supported? 'bottom(speed, 50)', # TODO: not supported? 'apercentile(speed, 10)', # TODO: TD-1316 - 'last_row(speed)', + # 'last_row(speed)', # TODO: commented out per TD-3231, we should re-create # Transformation Functions # 'diff(speed)', # TODO: no supported?! 'spread(speed)' diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 14f2b6e07e39322a3ce62f6176034fd2fba9edbc..b8552925b7e6ce710829b7df0a39ead1a25c17f7 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -183,6 +183,7 @@ python3 ./test.py -f query/isNullTest.py python3 ./test.py -f query/queryWithTaosdKilled.py python3 ./test.py -f query/floatCompare.py python3 ./test.py -f query/queryGroupbySort.py +python3 ./test.py -f query/queryBetweenAnd.py #stream python3 ./test.py -f stream/metric_1.py @@ -232,6 +233,7 @@ python3 ./test.py -f functions/function_top.py -r 1 python3 ./test.py -f functions/function_twa.py -r 1 python3 ./test.py -f functions/function_twa_test2.py python3 ./test.py -f functions/all_null_value.py +python3 ./test.py -f functions/function_percentile2.py python3 queryCount.py python3 ./test.py -f query/queryGroupbyWithInterval.py python3 client/twoClients.py diff --git a/tests/pytest/functions/function_percentile2.py b/tests/pytest/functions/function_percentile2.py new file mode 100644 index 0000000000000000000000000000000000000000..8a2b280c7df15f5626285671bf6f0432f730d10a --- /dev/null +++ b/tests/pytest/functions/function_percentile2.py @@ -0,0 +1,57 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 1000000 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + tdSql.execute("create table test(ts timestamp, col1 int, col2 float)") + for i in range(1000): + sql = "insert into test values" + batchSize = int (self.rowNum / 1000) + for j in range (batchSize): + currTime = self.ts + batchSize * i + j + sql += "(%d, 1, 2.37)" % currTime + tdSql.execute(sql) + + tdSql.query("select percentile(col1, 20) from test") + tdSql.checkData(0, 0, 1) + + tdSql.query("select percentile(col2, 20) from test") + tdSql.checkData(0, 0, 2.3699998) + + tdSql.query("select apercentile(col1, 20) from test") + tdSql.checkData(0, 0, 1) + + tdSql.query("select apercentile(col2, 20) from test") + tdSql.checkData(0, 0, 2.3699998) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_stddev.py b/tests/pytest/functions/function_stddev.py index 51e005936183853ee2a4c9cc58bd5dd192b99006..6e4ee694222247b884c2d40526d644e22cb2260f 100644 --- a/tests/pytest/functions/function_stddev.py +++ b/tests/pytest/functions/function_stddev.py @@ -26,6 +26,8 @@ class TDTestCase: self.rowNum = 10 self.ts = 1537146000000 + self.stb_prefix = 's' + self.subtb_prefix = 't' def run(self): tdSql.prepare() @@ -85,6 +87,33 @@ class TDTestCase: tdSql.query("select stddev(col6) from test1") tdSql.checkData(0, 0, np.std(floatData)) + + #add for td-3276 + sql="create table s (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool,c8 binary(20),c9 nchar(20),c11 int unsigned,c12 smallint unsigned,c13 tinyint unsigned,c14 bigint unsigned) \ + tags(t1 int, t2 float, t3 bigint, t4 smallint, t5 tinyint, t6 double, t7 bool,t8 binary(20),t9 nchar(20), t10 int unsigned , t11 smallint unsigned , t12 tinyint unsigned , t13 bigint unsigned)" + tdSql.execute(sql) + for j in range(2): + if j % 2 == 0: + sql = "create table %s using %s tags(NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL)" % \ + (self.subtb_prefix+str(j)+'_'+str(j),self.stb_prefix) + else: + sql = "create table %s using %s tags(%d,%d,%d,%d,%d,%d,%d,'%s','%s',%d,%d,%d,%d)" % \ + (self.subtb_prefix+str(j)+'_'+str(j),self.stb_prefix,j,j/2.0,j%41,j%51,j%53,j*1.0,j%2,'taos'+str(j),'涛思'+str(j), j%43, j%23 , j%17 , j%3167) + tdSql.execute(sql) + for i in range(10): + if i % 5 == 0 : + ret = tdSql.execute( + "insert into %s values (%d , NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL)" % + (self.subtb_prefix+str(j)+'_'+str(j), self.ts+i)) + else: + ret = tdSql.execute( + "insert into %s values (%d , %d,%d,%d,%d,%d,%d,%d,'%s','%s',%d,%d,%d,%d)" % + (self.subtb_prefix+str(j)+'_'+str(j), self.ts+i, i%100, i/2.0, i%41, i%51, i%53, i*1.0, i%2,'taos'+str(i),'涛思'+str(i), i%43, i%23 , i%17 , i%3167)) + + for i in range(13): + tdSql.query('select stddev(c4) from s group by t%s' % str(i+1) ) + + def stop(self): tdSql.close() diff --git a/tests/pytest/pytest_3.sh b/tests/pytest/pytest_3.sh index b1e2539fa935bcb9ce69482325e1e4c6df0503f2..a40cbb0d558e30943e29ea55f2274a96df2a3e6d 100755 --- a/tests/pytest/pytest_3.sh +++ b/tests/pytest/pytest_3.sh @@ -79,6 +79,7 @@ python3 test.py -f query/queryFillTest.py # tools python3 test.py -f tools/taosdemoTest.py +python3 test.py -f tools/taosdemoTestWithoutMetric.py python3 test.py -f tools/taosdumpTest.py python3 test.py -f tools/lowaTest.py #python3 test.py -f tools/taosdemoTest2.py diff --git a/tests/pytest/query/queryBetweenAnd.py b/tests/pytest/query/queryBetweenAnd.py new file mode 100644 index 0000000000000000000000000000000000000000..cd4320f5235222049f3c1dfc6a7bb8a1ab901b61 --- /dev/null +++ b/tests/pytest/query/queryBetweenAnd.py @@ -0,0 +1,206 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import taos +import sys + +from util.log import * +from util.sql import * +from util.cases import * + + + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + tdSql.execute( + '''create table if not exists supt + (ts timestamp, c1 int, c2 float, c3 bigint, c4 double, c5 smallint, c6 tinyint) + tags(location binary(64), type int, isused bool , family nchar(64))''' + ) + tdSql.execute("create table t1 using supt tags('beijing', 1, 1, '自行车')") + tdSql.execute("create table t2 using supt tags('shanghai', 2, 0, '拖拉机')") + + tdLog.printNoPrefix("==========step2:insert data") + for i in range(10): + tdSql.execute( + f"insert into t1 values (now+{i}m, {32767+i}, {20.0+i/10}, {2**31+i}, {3.4*10**38+i/10}, {127+i}, {i})" + ) + tdSql.execute( + f"insert into t2 values (now-{i}m, {-32767-i}, {20.0-i/10}, {-i-2**31}, {-i/10-3.4*10**38}, {-127-i}, {-i})" + ) + tdSql.execute( + f"insert into t1 values (now+11m, {2**31-1}, {pow(10,37)*34}, {pow(2,63)-1}, {1.7*10**308}, 32767, 127)" + ) + tdSql.execute( + f"insert into t2 values (now-11m, {1-2**31}, {-3.4*10**38}, {1-2**63}, {-1.7*10**308}, -32767, -127)" + ) + tdSql.execute( + f"insert into t2 values (now-12m, null , {-3.4*10**38}, null , {-1.7*10**308}, null , null)" + ) + + tdLog.printNoPrefix("==========step3:query timestamp type") + + tdSql.query("select * from t1 where ts between now-1m and now+10m") + tdSql.checkRows(10) + tdSql.query("select * from t1 where ts between '2021-01-01 00:00:00.000' and '2121-01-01 00:00:00.000'") + tdSql.checkRows(11) + tdSql.query("select * from t1 where ts between '1969-01-01 00:00:00.000' and '1969-12-31 23:59:59.999'") + tdSql.checkRows(0) + tdSql.query("select * from t1 where ts between -2793600 and 31507199") + tdSql.checkRows(0) + tdSql.query("select * from t1 where ts between 1609430400000 and 4765104000000") + tdSql.checkRows(11) + + tdLog.printNoPrefix("==========step4:query int type") + + tdSql.query("select * from t1 where c1 between 32767 and 32776") + tdSql.checkRows(10) + tdSql.query("select * from t1 where c1 between 32766.9 and 32776.1") + tdSql.checkRows(10) + tdSql.query("select * from t1 where c1 between 32776 and 32767") + tdSql.checkRows(0) + tdSql.error("select * from t1 where c1 between 'a' and 'e'") + # tdSql.query("select * from t1 where c1 between 0x64 and 0x69") + # tdSql.checkRows(6) + tdSql.error("select * from t1 where c1 not between 100 and 106") + tdSql.query(f"select * from t1 where c1 between {2**31-2} and {2**31+1}") + tdSql.checkRows(1) + tdSql.error(f"select * from t2 where c1 between null and {1-2**31}") + # tdSql.checkRows(3) + tdSql.query(f"select * from t2 where c1 between {-2**31} and {1-2**31}") + tdSql.checkRows(1) + + tdLog.printNoPrefix("==========step5:query float type") + + tdSql.query("select * from t1 where c2 between 20.0 and 21.0") + tdSql.checkRows(10) + tdSql.query(f"select * from t1 where c2 between {-3.4*10**38-1} and {3.4*10**38+1}") + tdSql.checkRows(11) + tdSql.query("select * from t1 where c2 between 21.0 and 20.0") + tdSql.checkRows(0) + tdSql.error("select * from t1 where c2 between 'DC3' and 'SYN'") + tdSql.error("select * from t1 where c2 not between 0.1 and 0.2") + # tdSql.query(f"select * from t1 where c2 between {pow(10,38)*3.4} and {pow(10,38)*3.4+1}") + # tdSql.checkRows(1) + tdSql.query(f"select * from t2 where c2 between {-3.4*10**38-1} and {-3.4*10**38}") + tdSql.checkRows(0) + tdSql.error(f"select * from t2 where c2 between null and {-3.4*10**38}") + # tdSql.checkRows(3) + + tdLog.printNoPrefix("==========step6:query bigint type") + + tdSql.query(f"select * from t1 where c3 between {2**31} and {2**31+10}") + tdSql.checkRows(10) + tdSql.error(f"select * from t1 where c3 between {-2**63} and {2**63}") + # tdSql.checkRows(11) + tdSql.query(f"select * from t1 where c3 between {2**31+10} and {2**31}") + tdSql.checkRows(0) + tdSql.error("select * from t1 where c3 between 'a' and 'z'") + tdSql.error("select * from t1 where c3 not between 1 and 2") + tdSql.query(f"select * from t1 where c3 between {2**63-2} and {2**63-1}") + tdSql.checkRows(1) + tdSql.error(f"select * from t2 where c3 between {-2**63} and {1-2**63}") + # tdSql.checkRows(3) + tdSql.error(f"select * from t2 where c3 between null and {1-2**63}") + # tdSql.checkRows(2) + + tdLog.printNoPrefix("==========step7:query double type") + + tdSql.query(f"select * from t1 where c4 between {3.4*10**38} and {3.4*10**38+10}") + tdSql.checkRows(10) + tdSql.query(f"select * from t1 where c4 between {1.7*10**308+1} and {1.7*10**308+2}") + # 因为精度原因，在超出bigint边界后，数值不能进行准确的判断 + # tdSql.checkRows(0) + tdSql.query(f"select * from t1 where c4 between {3.4*10**38+10} and {3.4*10**38}") + # tdSql.checkRows(0) + tdSql.error("select * from t1 where c4 between 'a' and 'z'") + tdSql.error("select * from t1 where c4 not between 1 and 2") + tdSql.query(f"select * from t1 where c4 between {1.7*10**308} and {1.7*10**308+1}") + tdSql.checkRows(1) + tdSql.query(f"select * from t2 where c4 between {-1.7*10**308-1} and {-1.7*10**308}") + # tdSql.checkRows(3) + tdSql.error(f"select * from t2 where c4 between null and {-1.7*10**308}") + # tdSql.checkRows(3) + + tdLog.printNoPrefix("==========step8:query smallint type") + + tdSql.query("select * from t1 where c5 between 127 and 136") + tdSql.checkRows(10) + tdSql.query("select * from t1 where c5 between 126.9 and 135.9") + tdSql.checkRows(9) + tdSql.query("select * from t1 where c5 between 136 and 127") + tdSql.checkRows(0) + tdSql.error("select * from t1 where c5 between '~' and 'ˆ'") + tdSql.error("select * from t1 where c5 not between 1 and 2") + tdSql.query("select * from t1 where c5 between 32767 and 32768") + tdSql.checkRows(1) + tdSql.query("select * from t2 where c5 between -32768 and -32767") + tdSql.checkRows(1) + tdSql.error("select * from t2 where c5 between null and -32767") + # tdSql.checkRows(1) + + tdLog.printNoPrefix("==========step9:query tinyint type") + + tdSql.query("select * from t1 where c6 between 0 and 9") + tdSql.checkRows(10) + tdSql.query("select * from t1 where c6 between -1.1 and 8.9") + tdSql.checkRows(9) + tdSql.query("select * from t1 where c6 between 9 and 0") + tdSql.checkRows(0) + tdSql.error("select * from t1 where c6 between 'NUL' and 'HT'") + tdSql.error("select * from t1 where c6 not between 1 and 2") + tdSql.query("select * from t1 where c6 between 127 and 128") + tdSql.checkRows(1) + tdSql.query("select * from t2 where c6 between -128 and -127") + tdSql.checkRows(1) + tdSql.error("select * from t2 where c6 between null and -127") + # tdSql.checkRows(3) + + tdLog.printNoPrefix("==========step10:invalid query type") + + tdSql.query("select * from supt where location between 'beijing' and 'shanghai'") + tdSql.checkRows(23) + # 非0值均解析为1，因此"between 负值 and o"解析为"between 1 and 0" + tdSql.query("select * from supt where isused between 0 and 1") + tdSql.checkRows(23) + tdSql.query("select * from supt where isused between -1 and 0") + tdSql.checkRows(0) + tdSql.error("select * from supt where isused between false and true") + tdSql.query("select * from supt where family between '拖拉机' and '自行车'") + tdSql.checkRows(23) + + tdLog.printNoPrefix("==========step11:query HEX/OCT/BIN type") + + tdSql.error("select * from t1 where c6 between 0x7f and 0x80") # check filter HEX + tdSql.error("select * from t1 where c6 between 0b1 and 0b11111") # check filter BIN + tdSql.error("select * from t1 where c6 between 0b1 and 0x80") + tdSql.error("select * from t1 where c6=0b1") + tdSql.error("select * from t1 where c6=0x1") + # 八进制数据会按照十进制数据进行判定 + tdSql.query("select * from t1 where c6 between 01 and 0200") # check filter OCT + tdSql.checkRows(10) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/tools/taosdemoTestWithoutMetric.py b/tests/pytest/tools/taosdemoTestWithoutMetric.py new file mode 100644 index 0000000000000000000000000000000000000000..647d6a37cb74e613950ccf72e9f2d99d85435c1a --- /dev/null +++ b/tests/pytest/tools/taosdemoTestWithoutMetric.py @@ -0,0 +1,72 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.numberOfTables = 100 + self.numberOfRecords = 1000 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath + "/build/bin/" + os.system("%staosdemo -y -t %d -n %d -x" % + (binPath, self.numberOfTables, self.numberOfRecords)) + + tdSql.query("show databases") + for i in range(18): + print(tdSql.getData(0, i) ) + tdSql.checkData(0, 2, self.numberOfTables) + + tdSql.execute("use test") + tdSql.query( + "select count(*) from test.t%d" % (self.numberOfTables -1)) + tdSql.checkData(0, 0, self.numberOfRecords) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 91c75efe9762a18042660423708e6dd9475166dd..6eaf4e18af53eac2b6e3c93749528993d8477e3d 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -136,7 +136,8 @@ class TDDnode: "monitorDebugFlag":"135", "udebugFlag":"135", "jnidebugFlag":"135", - "qdebugFlag":"135" + "qdebugFlag":"135", + "maxSQLLength":"1048576" } def init(self, path): diff --git a/tests/script/general/cache/new_metrics.sim b/tests/script/general/cache/new_metrics.sim index 00cbe843f955c618f8dfca56d9633eb911038d5e..eb9b042483251856c2ac7b3f973ab31176a8ff56 100644 --- a/tests/script/general/cache/new_metrics.sim +++ b/tests/script/general/cache/new_metrics.sim @@ -94,8 +94,9 @@ while $i < 10 $i = $i + 1 endw -print ==> sleep 8 seconds to renew cache -sleep 8000 +print ==> sleep 1 seconds to renew cache +sql reset query cache +sleep 1000 print =============== step5 diff --git a/tests/script/general/cache/restart_metrics.sim b/tests/script/general/cache/restart_metrics.sim index 4f2dae5fd95eba03de931234f6a71204948cb78a..a1b2365b2aef739d7bdf97ba82b757648285fe97 100644 --- a/tests/script/general/cache/restart_metrics.sim +++ b/tests/script/general/cache/restart_metrics.sim @@ -55,7 +55,7 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start print =============== step3 -print ==> sleep 8 seconds to renew cache +print ==> sleep 1 seconds to renew cache sql reset query cache sleep 1000 diff --git a/tests/script/general/cache/restart_table.sim b/tests/script/general/cache/restart_table.sim index 9b0d8f9c5b483b79da41fb4e86c17902d328a89d..1f7d982a2848a5df443f32a8c2b5c1edb97874d2 100644 --- a/tests/script/general/cache/restart_table.sim +++ b/tests/script/general/cache/restart_table.sim @@ -39,10 +39,9 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start print =============== step3 -print ==> sleep 8 seconds to renew cache -sleep 2000 +print ==> sleep 1 seconds to renew cache sql reset query cache -sleep 18000 +sleep 1000 print =============== step4 sql create database $db diff --git a/tests/script/general/db/topic1.sim b/tests/script/general/db/topic1.sim index e9e0bb5567372f79f5507f290ae29345eed22429..cdfb8ea2ec33a5322ab34de6d2688ef0dc1fc64f 100644 --- a/tests/script/general/db/topic1.sim +++ b/tests/script/general/db/topic1.sim @@ -740,11 +740,7 @@ endi if $data02 != 1 then return -1 endi -#numofvgroups -sql show t1.vgroups; -if $rows != 1 then - return -1 -endi + sql show t1.stables; if $rows != 1 then return -1 @@ -773,11 +769,8 @@ endi if $data02 != 2 then return -1 endi -#numofvgroups -sql show t1.vgroups; -if $rows != 2 then - return -1 -endi + + sql show t1.stables; if $rows != 1 then return -1 @@ -806,11 +799,7 @@ endi if $data02 != 3 then return -1 endi -#numofvgroups -sql show t1.vgroups; -if $rows != 3 then - return -1 -endi + sql show t1.stables; if $rows != 1 then return -1 diff --git a/tests/script/general/parser/col_arithmetic_operation.sim b/tests/script/general/parser/col_arithmetic_operation.sim index 7611bd582fc76d0e2c3fe2fe51bc2dcd74e7fbbe..8bb692e3bbe8af3ec9ed179ad29d40b4712d257b 100644 --- a/tests/script/general/parser/col_arithmetic_operation.sim +++ b/tests/script/general/parser/col_arithmetic_operation.sim @@ -93,6 +93,7 @@ $halfTbNum = $tbNum / 2 $nchar = 'nchar . $c $nchar = $nchar . ' + $ts = $ts + 1 sql insert into $tb5 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar ) $tb6 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar ) $tb7 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar ) $tb8 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar ) $tb9 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar ) $x = $x + 1 endw diff --git a/tests/script/general/parser/col_arithmetic_query.sim b/tests/script/general/parser/col_arithmetic_query.sim index 3f1a430e2bb5bab1112708a9925d74d27f030cc5..2c56c6445fa8134ab28837940a11ff6f4127b7c7 100644 --- a/tests/script/general/parser/col_arithmetic_query.sim +++ b/tests/script/general/parser/col_arithmetic_query.sim @@ -426,7 +426,7 @@ if $data02 != 9.000000020 then endi # all possible function in the arithmetic expression, add more -sql select min(c1) * max(c2) /4, sum(c1) * apercentile(c2, 20), apercentile(c4, 33) + 52/9, spread(c5)/min(c2), count(1)/sum(c1), avg(c2)*count(c2) from $stb where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-11-25 19:30:00.000'; +sql select min(c1) * max(c2) /4, sum(c1) * apercentile(c2, 20), apercentile(c4, 33) + 52/9, spread(c5)/min(c2), count(1)/sum(c1), avg(c2)*count(c2) from $stb where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-11-25 19:30:01.000'; if $rows != 1 then return -1 endi @@ -462,7 +462,7 @@ if $rows != 0 then endi # no result return [d.3] -sql select sum(c2) - avg(c2) from $stb where ts > '2018-11-25 19:30:00.000' +sql select sum(c2) - avg(c2) from $stb where ts > '2018-11-25 19:30:01.000' if $rows != 0 then return -1 endi @@ -520,35 +520,35 @@ if $data91 != 9 then endi # in group by column -sql select apercentile(c6, 50)-first(c6)+last(c5)*12, last(c5)*12 from ca_stb0 group by c2; -if $rows != 10 then - return -1 -endi - -if $data00 != 0.000000000 then - return -1 -endi - -if $data01 != 0.000000000 then - return -1 -endi - -if $data10 != 12.000000000 then - return -1 -endi - -if $data11 != 12.000000000 then - return -1 -endi - -if $data20 != 24.000000000 then - return -1 -endi - -if $data21 != 24.000000000 then - return -1 -endi - +#sql select apercentile(c6, 50)-first(c6)+last(c5)*12, last(c5)*12 from ca_stb0 group by c2; +#if $rows != 10 then +# return -1 +#endi +# +#if $data00 != 0.000000000 then +# return -1 +#endi +# +#if $data01 != 0.000000000 then +# return -1 +#endi +# +#if $data10 != 12.000000000 then +# return -1 +#endi +# +#if $data11 != 12.000000000 then +# return -1 +#endi +# +#if $data20 != 24.000000000 then +# return -1 +#endi +# +#if $data21 != 24.000000000 then +# return -1 +#endi +# sql_error select first(c6) - last(c6) *12 / count(*) from $stb group by c3; sql select first(c6) - last(c6) *12 / count(*) from $stb group by c5; diff --git a/tests/script/general/parser/fill.sim b/tests/script/general/parser/fill.sim index 7bf00145d619f2455a35686b9fd7424efbcc3f2a..d109dd50f7c0ba0684295dbf093ba1b280e04fce 100644 --- a/tests/script/general/parser/fill.sim +++ b/tests/script/general/parser/fill.sim @@ -313,6 +313,7 @@ if $rows != 9 then return -1 endi if $data01 != 0 then + print expect 0, actual:$data01 return -1 endi if $data11 != 6 then @@ -979,10 +980,6 @@ if $data00 != @20-01-01 01:01:00.000@ then return -1 endi -if $data00 != @20-01-01 01:01:00.000@ then - return -1 -endi -if $data1 if $data01 != 2.000000000 then return -1 endi diff --git a/tests/script/general/parser/function.sim b/tests/script/general/parser/function.sim index 8eb6aa55f2d06f4d90e14cd4035d2c460b6b1b00..9165b7e98e0510c3fead45acbc1baeceb0b653a3 100644 --- a/tests/script/general/parser/function.sim +++ b/tests/script/general/parser/function.sim @@ -775,8 +775,24 @@ if $rows != 1 then return -1 endi - if $data00 != 0.000000000 then return -1 endi +sql select count(tbname) from st1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +sql select count(id) from st1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim index a83c975be22e24684a50caccca4baa90bcae0b16..dd7331054c2ee6edd3ba07327057404dae7b3ed7 100644 --- a/tests/script/general/parser/groupby.sim +++ b/tests/script/general/parser/groupby.sim @@ -610,6 +610,11 @@ print =================>TD-2665 sql_error create table txx as select avg(c) as t from st; sql_error create table txx1 as select avg(c) as t from t1; +sql select stddev(c),stddev(c) from st group by c; +if $rows != 4 then + return -1 +endi + print =================>TD-2236 sql select first(ts),last(ts) from t1 group by c; if $rows != 4 then diff --git a/tests/script/general/parser/interp_test.sim b/tests/script/general/parser/interp_test.sim index 28601808306334286d2a9691626a2b1c1d211b30..81a77995fb828db00b65d085e0839e3b652385e0 100644 --- a/tests/script/general/parser/interp_test.sim +++ b/tests/script/general/parser/interp_test.sim @@ -579,7 +579,7 @@ $tb = $tbPrefix . 0 ## interp(*) from stb + group by + fill(none) $t = $ts0 + 1000 - sql select interp(*) from $stb where ts = $t fill(NULL) group by tbname +sql select interp(*) from $stb where ts = $t fill(NULL) group by tbname if $rows != $tbNum then return -1 endi diff --git a/tests/script/general/parser/join.sim b/tests/script/general/parser/join.sim index 392228e12122bcbcd0d336ec9dbf32a0cf9ef4c5..73f83f1c9e2af21e49099ffd73a6b1e9d040b2eb 100644 --- a/tests/script/general/parser/join.sim +++ b/tests/script/general/parser/join.sim @@ -347,6 +347,7 @@ $val = $rowNum + $rowNum print $val print $rows if $rows != $val then + print expect $val , actual:$rows return -1 endi diff --git a/tests/script/general/parser/last_groupby.sim b/tests/script/general/parser/last_groupby.sim new file mode 100644 index 0000000000000000000000000000000000000000..f993324cd1ccb6e1d74f71b3a0d9b47cf3452b48 --- /dev/null +++ b/tests/script/general/parser/last_groupby.sim @@ -0,0 +1,99 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 +system sh/exec.sh -n dnode1 -s start + +sleep 100 +sql connect +print ======================== dnode1 start + +$db = testdb + +sql create database $db +sql use $db + +sql create stable st2 (ts timestamp, f1 int, f2 float, f3 double, f4 bigint, f5 smallint, f6 tinyint, f7 bool, f8 binary(10), f9 nchar(10)) tags (id1 int, id2 float, id3 nchar(10), id4 double, id5 smallint, id6 bigint, id7 binary(10)) + +sql create table tb1 using st2 tags (1,1.0,"1",1.0,1,1,"1"); + +sql insert into tb1 values (now-200s,1,1.0,1.0,1,1,1,true,"1","1") +sql insert into tb1 values (now-100s,2,2.0,2.0,2,2,2,true,"2","2") +sql insert into tb1 values (now,3,3.0,3.0,3,3,3,true,"3","3") +sql insert into tb1 values (now+100s,4,4.0,4.0,4,4,4,true,"4","4") +sql insert into tb1 values (now+200s,4,4.0,4.0,4,4,4,true,"4","4") +sql insert into tb1 values (now+300s,4,4.0,4.0,4,4,4,true,"4","4") +sql insert into tb1 values (now+400s,4,4.0,4.0,4,4,4,true,"4","4") +sql insert into tb1 values (now+500s,4,4.0,4.0,4,4,4,true,"4","4") + +sql select f1,last(*) from st2 group by f1; + +if $rows != 4 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data02 != 1 then + print $data02 + return -1 +endi +if $data03 != 1.00000 then + return -1 +endi +if $data04 != 1.000000000 then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data06 != 1 then + return -1 +endi +if $data07 != 1 then + return -1 +endi +if $data08 != 1 then + return -1 +endi +if $data09 != 1 then + return -1 +endi + +sql select f1,last(f1,st2.*) from st2 group by f1; +if $rows != 4 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi +if $data03 != 1 then + return -1 +endi +if $data04 != 1.00000 then + return -1 +endi +if $data05 != 1.000000000 then + return -1 +endi +if $data06 != 1 then + return -1 +endi +if $data07 != 1 then + return -1 +endi +if $data08 != 1 then + return -1 +endi +if $data09 != 1 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/lastrow_query.sim b/tests/script/general/parser/lastrow_query.sim index a87b3cc646871e442cd3540fa163f1d5492c1456..3fd88cfc1bb4e651ae16801c249aa88f0a3797ba 100644 --- a/tests/script/general/parser/lastrow_query.sim +++ b/tests/script/general/parser/lastrow_query.sim @@ -16,6 +16,9 @@ $stb = $stbPrefix . $i sql use $db +print ========>TD-3231 last_row with group by column error +sql_error select last_row(c1) from $stb group by c1; + ##### select lastrow from STable with two vnodes, timestamp decreases from tables in vnode0 to tables in vnode1 sql select last_row(*) from $stb if $rows != 1 then @@ -224,4 +227,4 @@ sql create table tu(ts timestamp, k int) sql select last_row(*) from tu if $row != 0 then return -1 -endi \ No newline at end of file +endi diff --git a/tests/script/general/parser/limit1_stb.sim b/tests/script/general/parser/limit1_stb.sim index a0e3a45d2fc28002e2e64b195191b72659849a53..513e2fac026c0d1d617ba4126d22c01f82527ca0 100644 --- a/tests/script/general/parser/limit1_stb.sim +++ b/tests/script/general/parser/limit1_stb.sim @@ -538,6 +538,7 @@ $offset = $offset + 1 sql select max(c1), min(c2), avg(c3), count(c4), sum(c5), spread(c6), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 1 and t1 < 8 interval(5m) limit $offset offset $offset $val = $rowNum - $offset if $rows != $val then + print expect $val, actual:$rows return -1 endi if $data00 != @18-10-22 02:30:00.000@ then diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index 255389a2df12e470890ca74e9b3872a28bc119c3..c2ce5df12e9e4d60824950f21ac8afb9d64b5235 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -107,3 +107,6 @@ sleep 100 run general/parser/function.sim sleep 100 run general/parser/stableOp.sim +sleep 100 +run general/parser/slimit_alter_tags.sim + diff --git a/tests/script/general/parser/topbot.sim b/tests/script/general/parser/topbot.sim index 3e73e4967be505d80ebf47809e63c89bc2846a04..80a122238ec70136daa444b78764f5881563451d 100644 --- a/tests/script/general/parser/topbot.sim +++ b/tests/script/general/parser/topbot.sim @@ -73,6 +73,60 @@ if $row != 100 then return -1 endi +sql select bottom(c3, 5) from tb_tb1 interval(1y); +if $rows != 5 then + return -1 +endi + +if $data01 != 0.00000 then + print expect 0.00000, actual:$data01 + return -1 +endi + +if $data11 != 0.00000 then + return -1 +endi + +if $data21 != 0.00000 then + return -1 +endi + +if $data31 != 0.00000 then + return -1 +endi + +sql select top(c4, 5) from tb_tb1 interval(1y); +if $rows != 5 then + return -1 +endi + +if $data01 != 9.000000000 then + print expect 9.000000000, acutal:$data01 + return -1 +endi + +if $data11 != 9.000000000 then + return -1 +endi + +if $data21 != 9.000000000 then + return -1 +endi + +if $data31 != 9.000000000 then + return -1 +endi + +sql select top(c3, 5) from tb_tb1 interval(40h) +if $rows != 25 then + return -1 +endi + +if $data01 != 9.00000 then + print expect 9.00000, actual:$data01 + return -1 +endi + sql select last(*) from tb_tb9 if $row != 1 then return -1 diff --git a/tests/script/general/stream/restart_stream.sim b/tests/script/general/stream/restart_stream.sim index 5da69075361da7c2fad3ed9fde7384493fac9519..c8be10103d16a856fb0c083546cfaf8b939a1b98 100644 --- a/tests/script/general/stream/restart_stream.sim +++ b/tests/script/general/stream/restart_stream.sim @@ -100,8 +100,9 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start print =============== step5 -print ==> sleep 8 seconds to renew cache -sleep 8000 +print ==> renew cache +sql reset query cache +sleep 1000 print =============== step6 diff --git a/tests/script/jenkins/basic_1.txt b/tests/script/jenkins/basic_1.txt index 17245767344e50389d10bc997be33abb78c289ef..ae72e76b2f6c444e84fdaee3818200983796ace3 100644 --- a/tests/script/jenkins/basic_1.txt +++ b/tests/script/jenkins/basic_1.txt @@ -22,8 +22,6 @@ ./test.sh -f general/http/grafana_bug.sim ./test.sh -f general/http/grafana.sim - - ./test.sh -f general/insert/basic.sim ./test.sh -f general/insert/insert_drop.sim ./test.sh -f general/insert/query_block1_memory.sim