diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 37d05de731d88d6f7e8522349976221176b3199d..8d82c65cee5cb26c3a4a37e6bbe6076dae28aaa5 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -20,9 +20,6 @@ extern "C" { #endif -/* - * @date 2018/09/30 - */ #include "exception.h" #include "os.h" #include "qExtbuffer.h" @@ -216,7 +213,7 @@ STableMetaInfo* tscGetMetaInfo(SQueryInfo *pQueryInfo, int32_t tableIndex); SQueryInfo *tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex); SQueryInfo *tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex); -void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache); +void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo); STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables); @@ -276,7 +273,7 @@ void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex); bool hasMoreVnodesToTry(SSqlObj *pSql); bool hasMoreClauseToTry(SSqlObj* pSql); -void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeFromCache); +void tscFreeQueryInfo(SSqlCmd* pCmd); void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp); void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows); @@ -290,6 +287,14 @@ int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_ char* serializeTagData(STagData* pTagData, char* pMsg); int32_t copyTagData(STagData* dst, const STagData* src); +STableMeta* createSuperTableMeta(STableMetaMsg* pChild); +uint32_t tscGetTableMetaSize(STableMeta* pTableMeta); +CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta); +uint32_t tscGetTableMetaMaxSize(); +int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name); +STableMeta* tscTableMetaClone(STableMeta* pTableMeta); + + void* malloc_throw(size_t size); void* calloc_throw(size_t nmemb, size_t size); char* strdup_throw(const char* str); diff --git a/src/client/inc/tschemautil.h b/src/client/inc/tschemautil.h index f6dc45398f35c38598f3f3132b2f6f5601a4ed68..7c41164a046165c785d3d4e0e0a8979854e4b6f2 100644 --- a/src/client/inc/tschemautil.h +++ b/src/client/inc/tschemautil.h @@ -105,7 +105,10 @@ SSchema tscGetTbnameColumnSchema(); * @param size size of the table meta * @return */ -STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size); +STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg); + +bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src); +SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg); #ifdef __cplusplus } diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 68794701454281a6d10fda818b28a13a0f5d3e5a..25a299d0988144980dded10b170cbe7b7c662b04 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -56,23 +56,28 @@ typedef struct STableComInfo { int32_t rowSize; } STableComInfo; -typedef struct SCorVgroupInfo { - int32_t version; - int8_t inUse; - int8_t numOfEps; - SEpAddr1 epAddr[TSDB_MAX_REPLICA]; -} SCorVgroupInfo; +typedef struct SNewVgroupInfo { + int32_t vgId; + int8_t inUse; + int8_t numOfEps; + SEpAddrMsg ep[TSDB_MAX_REPLICA]; +} SNewVgroupInfo; + +typedef struct CChildTableMeta { + int32_t vgId; + STableId id; + uint8_t tableType; + char sTableName[TSDB_TABLE_FNAME_LEN]; +} CChildTableMeta; typedef struct STableMeta { - STableComInfo tableInfo; + int32_t vgId; + STableId id; uint8_t tableType; + char sTableName[TSDB_TABLE_FNAME_LEN]; int16_t sversion; int16_t tversion; - char sTableId[TSDB_TABLE_FNAME_LEN]; - int32_t vgId; - SCorVgroupInfo corVgroupInfo; - STableId id; -// union {int64_t stableUid; SSchema* schema;}; + STableComInfo tableInfo; SSchema schema[]; // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info } STableMeta; @@ -171,7 +176,7 @@ typedef struct SParamInfo { } SParamInfo; typedef struct STableDataBlocks { - char tableId[TSDB_TABLE_FNAME_LEN]; + char tableName[TSDB_TABLE_FNAME_LEN]; int8_t tsSource; // where does the UNIX timestamp come from, server or client bool ordered; // if current rows are ordered or not int64_t vgId; // virtual group id @@ -249,7 +254,7 @@ typedef struct { int8_t submitSchema; // submit block is built with table schema STagData tagData; // NOTE: pTagData->data is used as a variant length array - STableMeta **pTableMetaList; // all involved tableMeta list of current insert sql statement. + char **pTableNameList; // all involved tableMeta list of current insert sql statement. int32_t numOfTables; SHashObj *pTableBlockHashList; // data block for each table @@ -386,7 +391,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet); int tscProcessSql(SSqlObj *pSql); int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex); -void tscQueueAsyncRes(SSqlObj *pSql); +void tscAsyncResultOnError(SSqlObj *pSql); void tscQueueAsyncError(void(*fp), void *param, int32_t code); @@ -400,7 +405,7 @@ void tscRestoreSQLFuncForSTableQuery(SQueryInfo *pQueryInfo); int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo); void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo); -void tscResetSqlCmdObj(SSqlCmd *pCmd, bool removeFromCache); +void tscResetSqlCmdObj(SSqlCmd *pCmd); /** * free query result of the sql object @@ -414,7 +419,6 @@ void tscFreeSqlResult(SSqlObj *pSql); */ void tscFreeSqlObj(SSqlObj *pSql); void tscFreeRegisteredSqlObj(void *pSql); -void tscFreeTableMetaHelper(void *pTableMeta); void tscCloseTscObj(void *pObj); @@ -479,7 +483,9 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField } } -extern SCacheObj *tscMetaCache; +extern int32_t sentinel; +extern SHashObj *tscVgroupMap; +extern SHashObj *tscTableMetaInfo; extern int tscObjRef; extern void *tscTmr; diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index ad7041db10e489a43fb4b931424fc1f7fe678d31..96aeb9d60de1ab6fbaeebcb54e2da1ab316179f8 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -18,7 +18,6 @@ #include "tnote.h" #include "trpc.h" -#include "tcache.h" #include "tscLog.h" #include "tscSubquery.h" #include "tscLocalMerge.h" @@ -57,7 +56,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* para if (pSql->sqlstr == NULL) { tscError("%p failed to malloc sql string buffer", pSql); pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } @@ -71,7 +70,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* para if (code != TSDB_CODE_SUCCESS) { pSql->res.code = code; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } @@ -166,7 +165,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo pRes->code = numOfRows; } - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } @@ -217,7 +216,7 @@ void taos_fetch_rows_a(TAOS_RES *taosa, __async_cb_func_t fp, void *param) { pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE; pSql->param = param; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } @@ -280,7 +279,7 @@ void taos_fetch_row_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, TAOS_ROW), pSql->param = param; pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } @@ -382,7 +381,7 @@ void tscQueueAsyncError(void(*fp), void *param, int32_t code) { } -void tscQueueAsyncRes(SSqlObj *pSql) { +void tscAsyncResultOnError(SSqlObj *pSql) { if (pSql == NULL || pSql->signature != pSql) { tscDebug("%p SqlObj is freed, not add into queue async res", pSql); return; @@ -423,7 +422,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { // check if it is a sub-query of super table query first, if true, enter another routine if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, (TSDB_QUERY_TYPE_STABLE_SUBQUERY|TSDB_QUERY_TYPE_TAG_FILTER_QUERY))) { - tscDebug("%p update table meta in local cache, continue to process sql and send the corresponding query", pSql); + tscDebug("%p update local table meta, continue to process sql and send the corresponding query", pSql); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); code = tscGetTableMeta(pSql, pTableMetaInfo); @@ -440,7 +439,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { return; } else { // continue to process normal async query if (pCmd->parseFinished) { - tscDebug("%p update table meta in local cache, continue to process sql and send corresponding query", pSql); + tscDebug("%p update local table meta, continue to process sql and send corresponding query", pSql); STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); code = tscGetTableMeta(pSql, pTableMetaInfo); @@ -455,7 +454,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { if (pCmd->command == TSDB_SQL_SELECT) { tscDebug("%p redo parse sql string and proceed", pSql); pCmd->parseFinished = false; - tscResetSqlCmdObj(pCmd, false); + tscResetSqlCmdObj(pCmd); code = tsParseSql(pSql, true); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { @@ -532,6 +531,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { _error: if (code != TSDB_CODE_SUCCESS) { pSql->res.code = code; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } } diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 192af4dbdfea787a157ff2c03a0c4857128dede2..16bbd420c0cd77bd3f94ce64d840ccda36a36eb6 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -17,7 +17,6 @@ #include "taosmsg.h" #include "taosdef.h" -#include "tcache.h" #include "tname.h" #include "tscLog.h" #include "tscUtil.h" @@ -273,7 +272,7 @@ void tscSCreateCallBack(void *param, TAOS_RES *tres, int code) { if (pRes->code != TSDB_CODE_SUCCESS) { taos_free_result(pSql); free(builder); - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -291,7 +290,7 @@ void tscSCreateCallBack(void *param, TAOS_RES *tres, int code) { if (pRes->code == TSDB_CODE_SUCCESS) { (*pParentSql->fp)(pParentSql->param, pParentSql, code); } else { - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); } } } @@ -571,7 +570,7 @@ static int32_t tscRebuildDDLForSubTable(SSqlObj *pSql, const char *tableName, ch char fullName[TSDB_TABLE_FNAME_LEN * 2] = {0}; extractDBName(pTableMetaInfo->name, fullName); - extractTableName(pMeta->sTableId, param->sTableName); + extractTableName(pMeta->sTableName, param->sTableName); snprintf(fullName + strlen(fullName), TSDB_TABLE_FNAME_LEN - strlen(fullName), ".%s", param->sTableName); extractTableName(pTableMetaInfo->name, param->buf); @@ -901,7 +900,7 @@ int tscProcessLocalCmd(SSqlObj *pSql) { } else if (pCmd->command == TSDB_SQL_SHOW_CREATE_DATABASE) { pRes->code = tscProcessShowCreateDatabase(pSql); } else if (pCmd->command == TSDB_SQL_RESET_CACHE) { - taosCacheEmpty(tscMetaCache); + taosHashEmpty(tscTableMetaInfo); pRes->code = TSDB_CODE_SUCCESS; } else if (pCmd->command == TSDB_SQL_SERV_VERSION) { pRes->code = tscProcessServerVer(pSql); @@ -925,7 +924,7 @@ int tscProcessLocalCmd(SSqlObj *pSql) { (*pSql->fp)(pSql->param, pSql, code); } else if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS){ } else { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } return code; } diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 6f9bc6debc092557977803df6f589c89595a79ab..921aa9bade64759d149769726db0a5381c4b112c 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -89,7 +89,7 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc pCtx->startOffset = 0; pCtx->size = 1; pCtx->hasNull = true; - pCtx->currentStage = SECONDARY_STAGE_MERGE; + pCtx->currentStage = MERGE_STAGE; // for top/bottom function, the output of timestamp is the first column int32_t functionId = pExpr->functionId; @@ -1067,7 +1067,7 @@ static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, pCtx->param[0].i64Key = pExpr->param[0].i64Key; } - pCtx->currentStage = SECONDARY_STAGE_MERGE; + pCtx->currentStage = MERGE_STAGE; if (needInit) { aAggs[pCtx->functionId].init(pCtx); @@ -1080,7 +1080,7 @@ static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, continue; } - aAggs[functionId].distSecondaryMergeFunc(&pLocalReducer->pCtx[j]); + aAggs[functionId].mergeFunc(&pLocalReducer->pCtx[j]); } } @@ -1647,7 +1647,7 @@ int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_ // calculate the result from several other columns if (pSup->pArithExprInfo != NULL) { arithSup.pArithExpr = pSup->pArithExprInfo; - tExprTreeCalcTraverse(arithSup.pArithExpr->pExpr, (int32_t) pOutput->num, pbuf + pOutput->num*offset, &arithSup, TSDB_ORDER_ASC, getArithmeticInputSrc); + arithmeticTreeTraverse(arithSup.pArithExpr->pExpr, (int32_t) pOutput->num, pbuf + pOutput->num*offset, &arithSup, TSDB_ORDER_ASC, getArithmeticInputSrc); } else { SSqlExpr* pExpr = pSup->pSqlExpr; memcpy(pbuf + pOutput->num * offset, pExpr->offset * pOutput->num + pOutput->data, (size_t)(pExpr->resBytes * pOutput->num)); diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index a2dcb5d89ab2578c15f8ea02c3b2f7b4e024ec71..ec90d21394a5dce657652fb9e6c2ca81dd2f120f 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1339,7 +1339,7 @@ int tsParseSql(SSqlObj *pSql, bool initial) { if (sqlstr == NULL || pSql->parseRetry >= 1 || ret != TSDB_CODE_TSC_INVALID_SQL) { free(sqlstr); } else { - tscResetSqlCmdObj(pCmd, true); + tscResetSqlCmdObj(pCmd); free(pSql->sqlstr); pSql->sqlstr = sqlstr; pSql->parseRetry++; @@ -1351,7 +1351,7 @@ int tsParseSql(SSqlObj *pSql, bool initial) { SSqlInfo SQLInfo = qSQLParse(pSql->sqlstr); ret = tscToSQLCmd(pSql, &SQLInfo); if (ret == TSDB_CODE_TSC_INVALID_SQL && pSql->parseRetry == 0 && SQLInfo.type == TSDB_SQL_NULL) { - tscResetSqlCmdObj(pCmd, true); + tscResetSqlCmdObj(pCmd); pSql->parseRetry++; ret = tscToSQLCmd(pSql, &SQLInfo); } @@ -1429,7 +1429,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { fclose(fp); pParentSql->res.code = code; - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } while (0); } @@ -1451,7 +1451,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { int32_t count = 0; int32_t maxRows = 0; - tfree(pCmd->pTableMetaList); + tfree(pCmd->pTableNameList); pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); if (pCmd->pTableBlockHashList == NULL) { @@ -1500,7 +1500,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { code = doPackSendDataBlock(pSql, count, pTableDataBlock); if (code != TSDB_CODE_SUCCESS) { pParentSql->res.code = code; - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -1535,7 +1535,7 @@ void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql) { tscError("%p failed to open file %s to load data from file, code:%s", pSql, pCmd->payload, tstrerror(pSql->res.code)); tfree(pSupporter); - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index d105c577db1b2c8d21ec16286dd19c1ca722e721..4a5a7b09a2013fe76cd4abcbc2f0fb903758d935 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -910,7 +910,7 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableNa * that are corresponding to the old name for the new table name. */ if (strlen(oldName) > 0 && strncasecmp(oldName, pTableMetaInfo->name, tListLen(pTableMetaInfo->name)) != 0) { - tscClearTableMetaInfo(pTableMetaInfo, false); + tscClearTableMetaInfo(pTableMetaInfo); } return TSDB_CODE_SUCCESS; @@ -4019,6 +4019,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, if (pExpr->nSQLOptr == TK_LIKE) { char* str = taosStringBuilderGetResult(sb, NULL); pQueryInfo->tagCond.tbnameCond.cond = strdup(str); + pQueryInfo->tagCond.tbnameCond.len = (int32_t) strlen(str); return TSDB_CODE_SUCCESS; } @@ -4068,6 +4069,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, char* str = taosStringBuilderGetResult(&sb1, NULL); pQueryInfo->tagCond.tbnameCond.cond = strdup(str); + pQueryInfo->tagCond.tbnameCond.len = (int32_t) strlen(str); taosStringBuilderDestroy(&sb1); tfree(segments); @@ -6381,6 +6383,41 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { return TSDB_CODE_SUCCESS; } +static int32_t checkQueryRangeForFill(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { + const char* msg3 = "start(end) time of query range required or time range too large"; + + if (pQueryInfo->interval.interval == 0) { + return TSDB_CODE_SUCCESS; + } + + bool initialWindows = TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER); + if (initialWindows) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + + int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); + + int64_t intervalRange = 0; + if (pQueryInfo->interval.intervalUnit == 'n' || pQueryInfo->interval.intervalUnit == 'y') { + int64_t f = 1; + if (pQueryInfo->interval.intervalUnit == 'n') { + f = 30L * MILLISECOND_PER_DAY; + } else if (pQueryInfo->interval.intervalUnit == 'y') { + f = 365L * MILLISECOND_PER_DAY; + } + + intervalRange = pQueryInfo->interval.interval * f; + } else { + intervalRange = pQueryInfo->interval.interval; + } + // number of result is not greater than 10,000,000 + if ((timeRange == 0) || (timeRange / intervalRange) >= MAX_INTERVAL_TIME_WINDOW) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + + return TSDB_CODE_SUCCESS; +} + int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { assert(pQuerySql != NULL && (pQuerySql->from == NULL || taosArrayGetSize(pQuerySql->from) > 0)); @@ -6576,31 +6613,21 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { tscFieldInfoUpdateOffset(pQueryInfo); - /* - * fill options are set at the end position, when all columns are set properly - * the columns may be increased due to group by operation - */ if (pQuerySql->fillType != NULL) { if (pQueryInfo->interval.interval == 0 && (!tscIsPointInterpQuery(pQueryInfo))) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - if (pQueryInfo->interval.interval > 0) { - bool initialWindows = TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER); - if (initialWindows) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); - // number of result is not greater than 10,000,000 - if ((timeRange == 0) || (timeRange / pQueryInfo->interval.interval) > MAX_INTERVAL_TIME_WINDOW) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); - } + /* + * fill options are set at the end position, when all columns are set properly + * the columns may be increased due to group by operation + */ + if ((code = checkQueryRangeForFill(pCmd, pQueryInfo)) != TSDB_CODE_SUCCESS) { + return code; } - int32_t ret = parseFillClause(pCmd, pQueryInfo, pQuerySql); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + if ((code = parseFillClause(pCmd, pQueryInfo, pQuerySql)) != TSDB_CODE_SUCCESS) { + return code; } } diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index d77bb9990ca9b1248e8136f936f76d120a9b3ec1..b9d38e3ea76d9ba15c7fe5eb5906e205423aade5 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -130,19 +130,8 @@ SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId) { return NULL; } -static void tscInitCorVgroupInfo(SCorVgroupInfo *corVgroupInfo, SVgroupMsg *pVgroupMsg) { - corVgroupInfo->version = 0; - corVgroupInfo->inUse = 0; - corVgroupInfo->numOfEps = pVgroupMsg->numOfEps; - - for (int32_t i = 0; i < pVgroupMsg->numOfEps; i++) { - corVgroupInfo->epAddr[i].fqdn = strndup(pVgroupMsg->epAddr[i].fqdn, tListLen(pVgroupMsg->epAddr[0].fqdn)); - corVgroupInfo->epAddr[i].port = pVgroupMsg->epAddr[i].port; - } -} - -STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size) { - assert(pTableMetaMsg != NULL); +STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) { + assert(pTableMetaMsg != NULL && pTableMetaMsg->numOfColumns >= 2 && pTableMetaMsg->numOfTags >= 0); int32_t schemaSize = (pTableMetaMsg->numOfColumns + pTableMetaMsg->numOfTags) * sizeof(SSchema); STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + schemaSize); @@ -159,11 +148,9 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size pTableMeta->id.tid = pTableMetaMsg->tid; pTableMeta->id.uid = pTableMetaMsg->uid; - tscInitCorVgroupInfo(&pTableMeta->corVgroupInfo, &pTableMetaMsg->vgroup); - pTableMeta->sversion = pTableMetaMsg->sversion; pTableMeta->tversion = pTableMetaMsg->tversion; - tstrncpy(pTableMeta->sTableId, pTableMetaMsg->sTableId, TSDB_TABLE_FNAME_LEN); + tstrncpy(pTableMeta->sTableName, pTableMetaMsg->sTableName, TSDB_TABLE_FNAME_LEN); memcpy(pTableMeta->schema, pTableMetaMsg->schema, schemaSize); @@ -172,13 +159,44 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; } - if (size != NULL) { - *size = sizeof(STableMeta) + schemaSize; - } - return pTableMeta; } +bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src) { + assert(pExisted != NULL && src != NULL); + if (pExisted->numOfEps != src->numOfEps) { + return false; + } + + for(int32_t i = 0; i < pExisted->numOfEps; ++i) { + if (pExisted->ep[i].port != src->epAddr[i].port) { + return false; + } + + if (strncmp(pExisted->ep[i].fqdn, src->epAddr[i].fqdn, tListLen(pExisted->ep[i].fqdn)) != 0) { + return false; + } + } + + return true; +} + +SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg) { + assert(pVgroupMsg != NULL); + + SNewVgroupInfo info = {0}; + info.numOfEps = pVgroupMsg->numOfEps; + info.vgId = pVgroupMsg->vgId; + info.inUse = 0; + + for(int32_t i = 0; i < pVgroupMsg->numOfEps; ++i) { + tstrncpy(info.ep[i].fqdn, pVgroupMsg->epAddr[i].fqdn, TSDB_FQDN_LEN); + info.ep[i].port = pVgroupMsg->epAddr[i].port; + } + + return info; +} + // todo refactor UNUSED_FUNC static FORCE_INLINE char* skipSegments(char* input, char delim, int32_t num) { for (int32_t i = 0; i < num; ++i) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index f7a223626261d645687b32878f9f3c7b1fc72425..4949aa9b9dab6efb96f411f433ec420c6fd6d338 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -14,7 +14,6 @@ */ #include "os.h" -#include "tcache.h" #include "tcmdtype.h" #include "trpc.h" #include "tscLocalMerge.h" @@ -85,7 +84,8 @@ static void tscEpSetHtons(SRpcEpSet *s) { bool tscEpSetIsEqual(SRpcEpSet *s1, SRpcEpSet *s2) { if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) { return false; - } + } + for (int32_t i = 0; i < s1->numOfEps; i++) { if (s1->port[i] != s2->port[i] || strncmp(s1->fqdn[i], s2->fqdn[i], TSDB_FQDN_LEN) != 0) @@ -93,6 +93,7 @@ bool tscEpSetIsEqual(SRpcEpSet *s1, SRpcEpSet *s2) { } return true; } + void tscUpdateMgmtEpSet(SSqlObj *pSql, SRpcEpSet *pEpSet) { // no need to update if equal SRpcCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; @@ -101,37 +102,44 @@ void tscUpdateMgmtEpSet(SSqlObj *pSql, SRpcEpSet *pEpSet) { taosCorEndWrite(&pCorEpSet->version); } -static void tscDumpEpSetFromVgroupInfo(SRpcEpSet *pEpSet, SCorVgroupInfo *pVgroupInfo) { +static void tscDumpEpSetFromVgroupInfo(SRpcEpSet *pEpSet, SNewVgroupInfo *pVgroupInfo) { if (pVgroupInfo == NULL) { return;} - taosCorBeginRead(&pVgroupInfo->version); int8_t inUse = pVgroupInfo->inUse; pEpSet->inUse = (inUse >= 0 && inUse < TSDB_MAX_REPLICA) ? inUse: 0; pEpSet->numOfEps = pVgroupInfo->numOfEps; for (int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { - tstrncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, sizeof(pEpSet->fqdn[i])); - pEpSet->port[i] = pVgroupInfo->epAddr[i].port; + tstrncpy(pEpSet->fqdn[i], pVgroupInfo->ep[i].fqdn, sizeof(pEpSet->fqdn[i])); + pEpSet->port[i] = pVgroupInfo->ep[i].port; } - taosCorEndRead(&pVgroupInfo->version); } static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) { SSqlCmd *pCmd = &pObj->cmd; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { return;} - SCorVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->corVgroupInfo; + if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { + return; + } + + int32_t vgId = pTableMetaInfo->pTableMeta->vgId; + if (pTableMetaInfo->pTableMeta->tableType == TSDB_SUPER_TABLE) { + assert(vgId == 0); + return; + } + + SNewVgroupInfo vgroupInfo = {.vgId = -1}; + taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo)); + assert(vgroupInfo.numOfEps > 0 && vgroupInfo.vgId > 0); - taosCorBeginWrite(&pVgroupInfo->version); - tscDebug("before: Endpoint in use: %d", pVgroupInfo->inUse); - pVgroupInfo->inUse = pEpSet->inUse; - pVgroupInfo->numOfEps = pEpSet->numOfEps; - for (int32_t i = 0; i < pVgroupInfo->numOfEps; i++) { - tfree(pVgroupInfo->epAddr[i].fqdn); - pVgroupInfo->epAddr[i].fqdn = strndup(pEpSet->fqdn[i], tListLen(pEpSet->fqdn[i])); - pVgroupInfo->epAddr[i].port = pEpSet->port[i]; + tscDebug("before: Endpoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps); + vgroupInfo.inUse = pEpSet->inUse; + vgroupInfo.numOfEps = pEpSet->numOfEps; + for (int32_t i = 0; i < vgroupInfo.numOfEps; i++) { + strncpy(vgroupInfo.ep[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN); + vgroupInfo.ep[i].port = pEpSet->port[i]; } - tscDebug("after: EndPoint in use: %d", pVgroupInfo->inUse); - taosCorEndWrite(&pVgroupInfo->version); + tscDebug("after: EndPoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps); + taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(SNewVgroupInfo)); } void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { @@ -303,7 +311,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { return; } - if (pEpSet) { + if (pEpSet) { // todo update this if (!tscEpSetIsEqual(&pSql->epSet, pEpSet)) { if (pCmd->command < TSDB_SQL_MGMT) { tscUpdateVgroupInfo(pSql, pEpSet); @@ -437,7 +445,7 @@ int doProcessSql(SSqlObj *pSql) { } if (pRes->code != TSDB_CODE_SUCCESS) { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return pRes->code; } @@ -446,7 +454,7 @@ int doProcessSql(SSqlObj *pSql) { // NOTE: if code is TSDB_CODE_SUCCESS, pSql may have been released here already by other threads. if (code != TSDB_CODE_SUCCESS) { pRes->code = code; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return code; } @@ -549,7 +557,10 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // pSql->cmd.payloadLen is set during copying data into payload pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; - tscDumpEpSetFromVgroupInfo(&pSql->epSet, &pTableMeta->corVgroupInfo); + + SNewVgroupInfo vgroupInfo = {0}; + taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo)); + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); tscDebug("%p build submit msg, vgId:%d numOfTables:%d numberOfEP:%d", pSql, pTableMeta->vgId, pSql->cmd.numOfTablesInSubmit, pSql->epSet.numOfEps); @@ -559,9 +570,11 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { /* * for table query, simply return the size <= 1k */ -static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { +static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql, int32_t clauseIndex) { const static int32_t MIN_QUERY_MSG_PKT_SIZE = TSDB_MAX_BYTES_PER_ROW * 5; - SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); + + SSqlCmd* pCmd = &pSql->cmd; + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo)); @@ -569,6 +582,8 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { int32_t exprSize = (int32_t)(sizeof(SSqlFuncMsg) * numOfExprs * 2); int32_t tsBufSize = (pQueryInfo->tsBuf != NULL) ? pQueryInfo->tsBuf->fileSize : 0; + int32_t sqlLen = (int32_t) strlen(pSql->sqlstr) + 1; + int32_t tableSerialize = 0; STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -585,7 +600,7 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { } return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize + tsBufSize + - tableSerialize + 4096; + tableSerialize + sqlLen + 4096; } static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char *pMsg) { @@ -611,7 +626,10 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char tscDebug("%p query on stable, vgIndex:%d, numOfVgroups:%d", pSql, index, pTableMetaInfo->vgroupList->numOfVgroups); } else { vgId = pTableMeta->vgId; - tscDumpEpSetFromVgroupInfo(&pSql->epSet, &pTableMeta->corVgroupInfo); + + SNewVgroupInfo vgroupInfo = {0}; + taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo)); + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); } pSql->epSet.inUse = rand()%pSql->epSet.numOfEps; @@ -662,7 +680,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - int32_t size = tscEstimateQueryMsgSize(pCmd, pCmd->clauseIndex); + int32_t size = tscEstimateQueryMsgSize(pSql, pCmd->clauseIndex); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { tscError("%p failed to malloc for query msg", pSql); @@ -695,7 +713,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { tstrncpy(pQueryMsg->version, version, tListLen(pQueryMsg->version)); int32_t numOfTags = (int32_t)taosArrayGetSize(pTableMetaInfo->tagColList); - + int32_t sqlLen = (int32_t) strlen(pSql->sqlstr); + if (pQueryInfo->order.order == TSDB_ORDER_ASC) { pQueryMsg->window.skey = htobe64(pQueryInfo->window.skey); pQueryMsg->window.ekey = htobe64(pQueryInfo->window.ekey); @@ -718,10 +737,12 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->interval.offsetUnit = pQueryInfo->interval.offsetUnit; pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType); + pQueryMsg->tbnameCondLen = htonl(pQueryInfo->tagCond.tbnameCond.len); pQueryMsg->numOfTags = htonl(numOfTags); pQueryMsg->queryType = htonl(pQueryInfo->type); - pQueryMsg->vgroupLimit = htobe64(pQueryInfo->vgroupLimit); - + pQueryMsg->vgroupLimit = htobe64(pQueryInfo->vgroupLimit); + pQueryMsg->sqlstrLen = htonl(sqlLen); + size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo); pQueryMsg->numOfOutput = htons((int16_t)numOfOutput); // this is the stage one output column number @@ -954,13 +975,11 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pMsg += pCond->len; } } - - if (pQueryInfo->tagCond.tbnameCond.cond == NULL) { - *pMsg = 0; - pMsg++; - } else { - strcpy(pMsg, pQueryInfo->tagCond.tbnameCond.cond); - pMsg += strlen(pQueryInfo->tagCond.tbnameCond.cond) + 1; + + SCond* pCond = &pQueryInfo->tagCond.tbnameCond; + if (pCond->len > 0) { + strncpy(pMsg, pCond->cond, pCond->len); + pMsg += pCond->len; } // compressed ts block @@ -981,6 +1000,9 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks); } + memcpy(pMsg, pSql->sqlstr, sqlLen); + pMsg += sqlLen; + int32_t msgLen = (int32_t)(pMsg - pCmd->payload); tscDebug("%p msg built success, len:%d bytes", pSql, msgLen); @@ -1447,10 +1469,14 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) { SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload; pCmd->payloadLen = htonl(pUpdateMsg->head.contLen); - SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + STableMeta *pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; + + SNewVgroupInfo vgroupInfo = {.vgId = -1}; + taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo)); + assert(vgroupInfo.vgId > 0); - tscDumpEpSetFromVgroupInfo(&pSql->epSet, &pTableMetaInfo->pTableMeta->corVgroupInfo); + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); return TSDB_CODE_SUCCESS; } @@ -1516,7 +1542,7 @@ static int tscLocalResultCommonBuilder(SSqlObj *pSql, int32_t numOfRes) { if (code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, pSql->res.numOfRows); } else { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } } @@ -1545,7 +1571,7 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { int32_t code = pRes->code; if (pRes->code != TSDB_CODE_SUCCESS) { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return code; } @@ -1564,7 +1590,7 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { if (pRes->code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); } else { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } return code; @@ -1808,19 +1834,46 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { pSchema++; } - size_t size = 0; - STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size); + STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); - // todo add one more function: taosAddDataIfNotExists(); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); assert(pTableMetaInfo->pTableMeta == NULL); - pTableMetaInfo->pTableMeta = (STableMeta *) taosCachePut(tscMetaCache, pTableMetaInfo->name, - strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer * 1000); + if (pTableMeta->tableType == TSDB_CHILD_TABLE) { + // check if super table hashmap or not + int32_t len = (int32_t) strnlen(pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); - if (pTableMetaInfo->pTableMeta == NULL) { - free(pTableMeta); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + // super tableMeta data alreay exists, create it according to tableMeta and add it to hash map + STableMeta* pSupTableMeta = createSuperTableMeta(pMetaMsg); + + uint32_t size = tscGetTableMetaSize(pSupTableMeta); + int32_t code = taosHashPut(tscTableMetaInfo, pTableMeta->sTableName, len, pSupTableMeta, size); + assert(code == TSDB_CODE_SUCCESS); + + tfree(pSupTableMeta); + + CChildTableMeta* cMeta = tscCreateChildMeta(pTableMeta); + taosHashPut(tscTableMetaInfo, pTableMetaInfo->name, strlen(pTableMetaInfo->name), cMeta, sizeof(CChildTableMeta)); + tfree(cMeta); + } else { + uint32_t s = tscGetTableMetaSize(pTableMeta); + taosHashPut(tscTableMetaInfo, pTableMetaInfo->name, strlen(pTableMetaInfo->name), pTableMeta, s); + } + + // update the vgroupInfo if needed + if (pTableMeta->vgId > 0) { + int32_t vgId = pTableMeta->vgId; + assert(pTableMeta->tableType != TSDB_SUPER_TABLE); + + SNewVgroupInfo vgroupInfo = {.inUse = -1}; + taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo)); + + if (((vgroupInfo.inUse >= 0) && !vgroupInfoIdentical(&vgroupInfo, &pMetaMsg->vgroup)) || + (vgroupInfo.inUse < 0)) { // vgroup info exists, compare with it + vgroupInfo = createNewVgroupInfo(&pMetaMsg->vgroup); + taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(vgroupInfo)); + tscDebug("add new VgroupInfo, vgId:%d, total:%d", vgId, (int32_t) taosHashGetSize(tscVgroupMap)); + } } tscDebug("%p recv table meta, uid:%"PRId64 ", tid:%d, name:%s", pSql, pTableMeta->id.uid, pTableMeta->id.tid, pTableMetaInfo->name); @@ -1831,8 +1884,8 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { /** * multi table meta rsp pkg format: - * | STaosRsp | ieType | SMultiTableInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2 - * |...... 1B 1B 4B + * | STaosRsp | SMultiTableInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2 + * |...... 1B 4B **/ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) { #if 0 @@ -1986,14 +2039,10 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { return pSql->res.code; } -/* - * current process do not use the cache at all - */ int tscProcessShowRsp(SSqlObj *pSql) { STableMetaMsg *pMetaMsg; SShowRsp * pShow; SSchema * pSchema; - char key[20]; SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; @@ -2018,20 +2067,10 @@ int tscProcessShowRsp(SSqlObj *pSql) { pSchema++; } - key[0] = pCmd->msgType + 'a'; - strcpy(key + 1, "showlist"); + tfree(pTableMetaInfo->pTableMeta); + pTableMetaInfo->pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); - if (pTableMetaInfo->pTableMeta != NULL) { - taosCacheRelease(tscMetaCache, (void *)&(pTableMetaInfo->pTableMeta), false); - } - - size_t size = 0; - STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size); - - pTableMetaInfo->pTableMeta = taosCachePut(tscMetaCache, key, strlen(key), (char *)pTableMeta, size, - tsTableMetaKeepTimer * 1000); SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - if (pQueryInfo->colList == NULL) { pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); } @@ -2054,12 +2093,9 @@ int tscProcessShowRsp(SSqlObj *pSql) { pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; tscFieldInfoUpdateOffset(pQueryInfo); - - tfree(pTableMeta); return 0; } -// TODO multithread problem static void createHBObj(STscObj* pObj) { if (pObj->hbrid != 0) { return; @@ -2141,51 +2177,34 @@ int tscProcessUseDbRsp(SSqlObj *pSql) { int tscProcessDropDbRsp(SSqlObj *pSql) { pSql->pTscObj->db[0] = 0; - taosCacheEmpty(tscMetaCache); + taosHashEmpty(tscTableMetaInfo); return 0; } int tscProcessDropTableRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - STableMeta *pTableMeta = taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); - if (pTableMeta == NULL) { /* not in cache, abort */ - return 0; - } + //The cached tableMeta is expired in this case, so clean it in hash table + taosHashRemove(tscTableMetaInfo, pTableMetaInfo->name, strnlen(pTableMetaInfo->name, TSDB_TABLE_FNAME_LEN)); + tscDebug("%p remove table meta after drop table:%s, numOfRemain:%d", pSql, pTableMetaInfo->name, + (int32_t) taosHashGetSize(tscTableMetaInfo)); - /* - * 1. if a user drops one table, which is the only table in a vnode, remove operation will incur vnode to be removed. - * 2. Then, a user creates a new metric followed by a table with identical name of removed table but different schema, - * here the table will reside in a new vnode. - * The cached information is expired, however, we may have lost the ref of original meter. So, clear whole cache - * instead. - */ - tscDebug("%p force release table meta after drop table:%s", pSql, pTableMetaInfo->name); - taosCacheRelease(tscMetaCache, (void **)&pTableMeta, true); assert(pTableMetaInfo->pTableMeta == NULL); - return 0; } int tscProcessAlterTableMsgRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - STableMeta *pTableMeta = taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); - if (pTableMeta == NULL) { /* not in cache, abort */ - return 0; - } + char* name = pTableMetaInfo->name; + tscDebug("%p remove tableMeta in hashMap after alter-table: %s", pSql, name); - tscDebug("%p force release metermeta in cache after alter-table: %s", pSql, pTableMetaInfo->name); - taosCacheRelease(tscMetaCache, (void **)&pTableMeta, true); + bool isSuperTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); + tfree(pTableMetaInfo->pTableMeta); - if (pTableMetaInfo->pTableMeta) { - bool isSuperTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), true); - - if (isSuperTable) { // if it is a super table, reset whole query cache - tscDebug("%p reset query cache since table:%s is stable", pSql, pTableMetaInfo->name); - taosCacheEmpty(tscMetaCache); - } + if (isSuperTable) { // if it is a super table, iterate the hashTable and remove all the childTableMeta + taosHashEmpty(tscTableMetaInfo); } return 0; @@ -2195,6 +2214,7 @@ int tscProcessAlterDbMsgRsp(SSqlObj *pSql) { UNUSED(pSql); return 0; } + int tscProcessShowCreateRsp(SSqlObj *pSql) { return tscLocalResultCommonBuilder(pSql, 1); } @@ -2315,7 +2335,7 @@ static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaIn int32_t code = tscProcessSql(pNew); if (code == TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify upper application that current process need to be terminated + code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated } return code; @@ -2323,21 +2343,29 @@ static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaIn int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { assert(strlen(pTableMetaInfo->name) != 0); + tfree(pTableMetaInfo->pTableMeta); - // If this STableMetaInfo owns a table meta, release it first - if (pTableMetaInfo->pTableMeta != NULL) { - taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), false); - } - - pTableMetaInfo->pTableMeta = (STableMeta *)taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); - if (pTableMetaInfo->pTableMeta != NULL) { - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - tscDebug("%p retrieve table Meta from cache, the number of columns:%d, numOfTags:%d, %p", pSql, tinfo.numOfColumns, - tinfo.numOfTags, pTableMetaInfo->pTableMeta); + uint32_t size = tscGetTableMetaMaxSize(); + pTableMetaInfo->pTableMeta = calloc(1, size); + + pTableMetaInfo->pTableMeta->tableInfo.numOfColumns = -1; + int32_t len = (int32_t) strlen(pTableMetaInfo->name); + + taosHashGetClone(tscTableMetaInfo, pTableMetaInfo->name, len, NULL, pTableMetaInfo->pTableMeta, -1); + + // TODO resize the tableMeta + STableMeta* pMeta = pTableMetaInfo->pTableMeta; + if (pMeta->id.uid > 0) { + if (pMeta->tableType == TSDB_CHILD_TABLE) { + int32_t code = tscCreateTableMetaFromCChildMeta(pTableMetaInfo->pTableMeta, pTableMetaInfo->name); + if (code != TSDB_CODE_SUCCESS) { + return getTableMetaFromMnode(pSql, pTableMetaInfo); + } + } return TSDB_CODE_SUCCESS; } - + return getTableMetaFromMnode(pSql, pTableMetaInfo); } @@ -2347,7 +2375,7 @@ int tscGetTableMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool create } /** - * retrieve table meta from mnode, and update the local table meta cache. + * retrieve table meta from mnode, and update the local table meta hashmap. * @param pSql sql object * @param tableIndex table index * @return status code @@ -2355,16 +2383,18 @@ int tscGetTableMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool create int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) { SSqlCmd *pCmd = &pSql->cmd; - SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); + const char* name = pTableMetaInfo->name; STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - if (pTableMetaInfo->pTableMeta) { - tscDebug("%p update table meta, old meta numOfTags:%d, numOfCols:%d, uid:%" PRId64 ", addr:%p", pSql, - tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid, pTableMeta); + if (pTableMeta) { + tscDebug("%p update table meta:%s, old meta numOfTags:%d, numOfCols:%d, uid:%" PRId64, pSql, name, + tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid); } - taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), true); + // remove stored tableMeta info in hash table + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); return getTableMetaFromMnode(pSql, pTableMetaInfo); } @@ -2405,7 +2435,7 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo *pMInfo = tscGetMetaInfo(pQueryInfo, i); - STableMeta *pTableMeta = taosCacheAcquireByData(tscMetaCache, pMInfo->pTableMeta); + STableMeta* pTableMeta = tscTableMetaClone(pMInfo->pTableMeta); tscAddTableMetaInfo(pNewQueryInfo, pMInfo->name, pTableMeta, NULL, pMInfo->tagColList, pMInfo->pVgroupTables); } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index fa7bc99a9f304c45db46891f444509ec4c72e97f..377cb24b1d5b1c92c702cd113c4feea185617911 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -709,7 +709,7 @@ static void tscKillSTableQuery(SSqlObj *pSql) { pSubObj->rpcRid = -1; } - tscQueueAsyncRes(pSubObj); + tscAsyncResultOnError(pSubObj); taosReleaseRef(tscObjRef, pSubObj->self); } @@ -745,7 +745,7 @@ void taos_stop_query(TAOS_RES *res) { pSql->rpcRid = -1; } - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } } @@ -909,7 +909,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) { static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t tblListLen) { // must before clean the sqlcmd object - tscResetSqlCmdObj(&pSql->cmd, false); + tscResetSqlCmdObj(&pSql->cmd); SSqlCmd *pCmd = &pSql->cmd; diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 74b8e4d95878d8a784d4d82e28028e94bd32207b..c1ed9b0ba09f956207eeeccb2c7b8123538a2151 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -167,7 +167,9 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf retryDelay); STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0, 0); - taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), true); + + char* name = pTableMetaInfo->name; + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList); tscSetRetryTimer(pStream, pStream->pSql, retryDelay); @@ -269,9 +271,8 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf tscDebug("%p stream:%p, query on:%s, fetch result completed, fetched rows:%" PRId64, pSql, pStream, pTableMetaInfo->name, pStream->numOfRes); - // release the metric/meter meta information reference, so data in cache can be updated + tfree(pTableMetaInfo->pTableMeta); - taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false); tscFreeSqlResult(pSql); tfree(pSql->pSubs); pSql->subState.numOfSub = 0; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index a328ae4d0485690d5dc392a1b9ad5e33a1b930f1..681291d0db7039af09147d86f6db1fa48ccb233c 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -779,7 +779,7 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow pParentSql->res.code = numOfRows; quitAllSubquery(pParentSql, pSupporter); - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -796,7 +796,7 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow pParentSql->res.code = TAOS_SYSTEM_ERROR(errno); quitAllSubquery(pParentSql, pSupporter); - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -845,7 +845,7 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow if (code != TSDB_CODE_SUCCESS) { freeJoinSubqueryObj(pParentSql); pParentSql->res.code = code; - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); taosArrayDestroy(s1); taosArrayDestroy(s2); @@ -916,7 +916,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow pParentSql->res.code = numOfRows; quitAllSubquery(pParentSql, pSupporter); - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -930,7 +930,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow tscError("%p invalid ts comp file from vnode, abort subquery, file size:%d", pSql, numOfRows); pParentSql->res.code = TAOS_SYSTEM_ERROR(errno); - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -1028,7 +1028,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR pParentSql->res.code = numOfRows; tscError("%p retrieve failed, index:%d, code:%s", pSql, pSupporter->subqueryIndex, tstrerror(numOfRows)); - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -1155,7 +1155,7 @@ void tscFetchDatablockForSubquery(SSqlObj* pSql) { if (pSql->res.code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, 0); } else { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } return; @@ -1233,7 +1233,7 @@ void tscFetchDatablockForSubquery(SSqlObj* pSql) { if (pSql->res.code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, 0); } else { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } return; @@ -1344,7 +1344,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { if (pParentSql->res.code != TSDB_CODE_SUCCESS) { tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, code, pParentSql->res.code); quitAllSubquery(pParentSql, pSupporter); - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -1357,7 +1357,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { pParentSql->res.code = code; quitAllSubquery(pParentSql, pSupporter); - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -1403,7 +1403,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { if (pParentSql->res.code == TSDB_CODE_SUCCESS) { (*pParentSql->fp)(pParentSql->param, pParentSql, 0); } else { - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); } } } @@ -1612,7 +1612,7 @@ void tscHandleMasterJoinQuery(SSqlObj* pSql) { _error: pRes->code = code; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } static void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs) { @@ -1666,7 +1666,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, &pFinalModel, nBufferSize); if (ret != 0) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); tfree(pMemoryBuf); return ret; } @@ -1680,7 +1680,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pFinalModel,pState->numOfSub); - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return ret; } @@ -1890,7 +1890,7 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO (*pParentSql->fp)(pParentSql->param, pParentSql, pParentSql->res.code); } else { // regular super table query if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); } } } @@ -1968,7 +1968,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p if (pParentSql->res.code == TSDB_CODE_SUCCESS) { (*pParentSql->fp)(pParentSql->param, pParentSql, 0); } else { - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); } } @@ -2220,7 +2220,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) (*pParentObj->fp)(pParentObj->param, pParentObj, v); } else { if (!needRetryInsert(pParentObj, numOfSub)) { - tscQueueAsyncRes(pParentObj); + tscAsyncResultOnError(pParentObj); return; } @@ -2231,7 +2231,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) numOfFailed += 1; // clean up tableMeta in cache - tscFreeQueryInfo(&pSql->cmd, true); + tscFreeQueryInfo(&pSql->cmd); SQueryInfo* pQueryInfo = tscGetQueryInfoDetailSafely(&pSql->cmd, 0); STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pParentObj->cmd, pSql->cmd.clauseIndex, 0); tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL); @@ -2243,15 +2243,16 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) tscError("%p Async insertion completed, total inserted:%d rows, numOfFailed:%d, numOfTotal:%d", pParentObj, pParentObj->res.numOfRows, numOfFailed, numOfSub); - tscDebug("%p cleanup %d tableMeta in cache", pParentObj, pParentObj->cmd.numOfTables); + tscDebug("%p cleanup %d tableMeta in hashTable", pParentObj, pParentObj->cmd.numOfTables); for(int32_t i = 0; i < pParentObj->cmd.numOfTables; ++i) { - taosCacheRelease(tscMetaCache, (void**)&(pParentObj->cmd.pTableMetaList[i]), true); + char* name = pParentObj->cmd.pTableNameList[i]; + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); } pParentObj->cmd.parseFinished = false; pParentObj->subState.numOfRemain = numOfFailed; - tscResetSqlCmdObj(&pParentObj->cmd, false); + tscResetSqlCmdObj(&pParentObj->cmd); // in case of insert, redo parsing the sql string and build new submit data block for two reasons: // 1. the table Id(tid & uid) may have been update, the submit block needs to be updated accordingly. @@ -2264,7 +2265,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) if (code != TSDB_CODE_SUCCESS) { pParentObj->res.code = code; - tscQueueAsyncRes(pParentObj); + tscAsyncResultOnError(pParentObj); return; } @@ -2288,7 +2289,7 @@ int32_t tscHandleInsertRetry(SSqlObj* pParent, SSqlObj* pSql) { int32_t code = tscCopyDataBlockToPayload(pSql, pTableDataBlock); if ((pRes->code = code)!= TSDB_CODE_SUCCESS) { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return code; // here the pSql may have been released already. } @@ -2481,7 +2482,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { SSqlRes* pRes = &pSql->res; if (pRes->code != TSDB_CODE_SUCCESS) { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } @@ -2496,7 +2497,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { if (pRes->tsrow == NULL || pRes->buffer == NULL || pRes->length == NULL) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } @@ -2508,7 +2509,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { if (pRes->code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); } else { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } } diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index ff605dad72a876eacd4243b02be85bb6d8803634..dd0f248b85452342fb567357ba9346ab71754dfc 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -31,15 +31,20 @@ #include "tlocale.h" // global, not configurable -SCacheObj *tscMetaCache; // table meta cache -SHashObj *tscHashMap; // hash map to keep the global vgroup info -int tscObjRef = -1; -void *tscTmr; -void *tscQhandle; -void *tscCheckDiskUsageTmr; -int tscRefId = -1; -int tscNumOfObj = 0; // number of sqlObj in current process. +#define TSC_VAR_NOT_RELEASE 1 +#define TSC_VAR_RELEASED 0 +int32_t sentinel = TSC_VAR_NOT_RELEASE; + +SHashObj *tscVgroupMap; // hash map to keep the global vgroup info +SHashObj *tscTableMetaInfo; // table meta info +int32_t tscObjRef = -1; +void *tscTmr; +void *tscQhandle; +int32_t tscRefId = -1; +int32_t tscNumOfObj = 0; // number of sqlObj in current process. + +static void *tscCheckDiskUsageTmr; static pthread_once_t tscinit = PTHREAD_ONCE_INIT; void tscCheckDiskUsage(void *UNUSED_PARAM(para), void* UNUSED_PARAM(param)) { @@ -129,11 +134,11 @@ void taos_init_imp(void) { taosTmrReset(tscCheckDiskUsage, 10, NULL, tscTmr, &tscCheckDiskUsageTmr); } - int64_t refreshTime = 10; // 10 seconds by default - if (tscMetaCache == NULL) { - tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, tscFreeTableMetaHelper, "tableMeta"); + if (tscTableMetaInfo == NULL) { tscObjRef = taosOpenRef(40960, tscFreeRegisteredSqlObj); - tscHashMap = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + tscVgroupMap = taosHashInit(256, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + tscTableMetaInfo = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + tscDebug("TableMeta:%p", tscTableMetaInfo); } tscRefId = taosOpenRef(200, tscCloseTscObj); @@ -151,30 +156,38 @@ void taos_init() { pthread_once(&tscinit, taos_init_imp); } void taos_cleanup(void) { tscDebug("start to cleanup client environment"); - void* m = tscMetaCache; - if (m != NULL && atomic_val_compare_exchange_ptr(&tscMetaCache, m, 0) == m) { - taosCacheCleanup(m); + if (atomic_val_compare_exchange_32(&sentinel, TSC_VAR_NOT_RELEASE, TSC_VAR_RELEASED) != TSC_VAR_NOT_RELEASE) { + return; } - int refId = atomic_exchange_32(&tscObjRef, -1); - if (refId != -1) { - taosCloseRef(refId); - } + taosHashCleanup(tscTableMetaInfo); + tscTableMetaInfo = NULL; - m = tscQhandle; - if (m != NULL && atomic_val_compare_exchange_ptr(&tscQhandle, m, 0) == m) { - taosCleanUpScheduler(m); - } + taosHashCleanup(tscVgroupMap); + tscVgroupMap = NULL; + + int32_t id = tscObjRef; + tscObjRef = -1; + taosCloseRef(id); + + void* p = tscQhandle; + tscQhandle = NULL; + taosCleanUpScheduler(p); + + id = tscRefId; + tscRefId = -1; + taosCloseRef(id); - taosCloseRef(tscRefId); taosCleanupKeywordsTable(); taosCloseLog(); - if (tscEmbedded == 0) rpcCleanup(); - m = tscTmr; - if (m != NULL && atomic_val_compare_exchange_ptr(&tscTmr, m, 0) == m) { - taosTmrCleanUp(m); + if (tscEmbedded == 0) { + rpcCleanup(); } + + p = tscTmr; + tscTmr = NULL; + taosTmrCleanUp(p); } static int taos_options_imp(TSDB_OPTION option, const char *pStr) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 52e26fe95a627515d28080ff655f1c55e0674c82..a6e33778cde0a43f4cdd003e718ef49b89a41866 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -18,7 +18,6 @@ #include "os.h" #include "qAst.h" #include "taosmsg.h" -#include "tcache.h" #include "tkey.h" #include "tmd5.h" #include "tscLocalMerge.h" @@ -31,7 +30,7 @@ #include "ttokendef.h" static void freeQueryInfoImpl(SQueryInfo* pQueryInfo); -static void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache); +static void clearAllTableMetaInfo(SQueryInfo* pQueryInfo); SCond* tsGetSTableQueryCond(STagCond* pTagCond, uint64_t uid) { if (pTagCond->pCond == NULL) { @@ -379,17 +378,16 @@ static void tscDestroyResPointerInfo(SSqlRes* pRes) { pRes->data = NULL; // pRes->data points to the buffer of pRsp, no need to free } -void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeFromCache) { +void tscFreeQueryInfo(SSqlCmd* pCmd) { if (pCmd == NULL || pCmd->numOfClause == 0) { return; } for (int32_t i = 0; i < pCmd->numOfClause; ++i) { - char* addr = (char*)pCmd - offsetof(SSqlObj, cmd); SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, i); freeQueryInfoImpl(pQueryInfo); - clearAllTableMetaInfo(pQueryInfo, (const char*)addr, removeFromCache); + clearAllTableMetaInfo(pQueryInfo); tfree(pQueryInfo); } @@ -397,7 +395,7 @@ void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeFromCache) { tfree(pCmd->pQueryInfo); } -void tscResetSqlCmdObj(SSqlCmd* pCmd, bool removeFromCache) { +void tscResetSqlCmdObj(SSqlCmd* pCmd) { pCmd->command = 0; pCmd->numOfCols = 0; pCmd->count = 0; @@ -407,17 +405,17 @@ void tscResetSqlCmdObj(SSqlCmd* pCmd, bool removeFromCache) { pCmd->autoCreated = 0; for(int32_t i = 0; i < pCmd->numOfTables; ++i) { - if (pCmd->pTableMetaList && pCmd->pTableMetaList[i]) { - taosCacheRelease(tscMetaCache, (void**)&(pCmd->pTableMetaList[i]), false); + if (pCmd->pTableNameList && pCmd->pTableNameList[i]) { + tfree(pCmd->pTableNameList[i]); } } pCmd->numOfTables = 0; - tfree(pCmd->pTableMetaList); + tfree(pCmd->pTableNameList); pCmd->pTableBlockHashList = tscDestroyBlockHashTable(pCmd->pTableBlockHashList); pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); - tscFreeQueryInfo(pCmd, removeFromCache); + tscFreeQueryInfo(pCmd); } void tscFreeSqlResult(SSqlObj* pSql) { @@ -468,17 +466,6 @@ void tscFreeRegisteredSqlObj(void *pSql) { tscDebug("%p free SqlObj, total in tscObj:%d, total:%d", pSql, num, total); } -void tscFreeTableMetaHelper(void *pTableMeta) { - STableMeta* p = (STableMeta*) pTableMeta; - - int32_t numOfEps1 = p->corVgroupInfo.numOfEps; - assert(numOfEps1 >= 0 && numOfEps1 <= TSDB_MAX_REPLICA); - - for(int32_t i = 0; i < numOfEps1; ++i) { - tfree(p->corVgroupInfo.epAddr[i].fqdn); - } -} - void tscFreeSqlObj(SSqlObj* pSql) { if (pSql == NULL || pSql->signature != pSql) { return; @@ -506,7 +493,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { pSql->self = 0; tscFreeSqlResult(pSql); - tscResetSqlCmdObj(pCmd, false); + tscResetSqlCmdObj(pCmd); tfree(pCmd->tagData.data); pCmd->tagData.dataLen = 0; @@ -529,7 +516,7 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { // free the refcount for metermeta if (pDataBlock->pTableMeta != NULL) { - taosCacheRelease(tscMetaCache, (void**)&(pDataBlock->pTableMeta), false); + tfree(pDataBlock->pTableMeta); } tfree(pDataBlock); @@ -600,15 +587,15 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { // set the correct table meta object, the table meta has been locked in pDataBlocks, so it must be in the cache if (pTableMetaInfo->pTableMeta != pDataBlock->pTableMeta) { - tstrncpy(pTableMetaInfo->name, pDataBlock->tableId, sizeof(pTableMetaInfo->name)); + tstrncpy(pTableMetaInfo->name, pDataBlock->tableName, sizeof(pTableMetaInfo->name)); if (pTableMetaInfo->pTableMeta != NULL) { - taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false); + tfree(pTableMetaInfo->pTableMeta); } - pTableMetaInfo->pTableMeta = taosCacheTransfer(tscMetaCache, (void**)&pDataBlock->pTableMeta); + pTableMetaInfo->pTableMeta = tscTableMetaClone(pDataBlock->pTableMeta); } else { - assert(strncmp(pTableMetaInfo->name, pDataBlock->tableId, tListLen(pDataBlock->tableId)) == 0); + assert(strncmp(pTableMetaInfo->name, pDataBlock->tableName, tListLen(pDataBlock->tableName)) == 0); } /* @@ -671,14 +658,10 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff dataBuf->size = startOffset; dataBuf->tsSource = -1; - tstrncpy(dataBuf->tableId, name, sizeof(dataBuf->tableId)); + tstrncpy(dataBuf->tableName, name, sizeof(dataBuf->tableName)); - /* - * The table meta may be released since the table meta cache are completed clean by other thread - * due to operation such as drop database. So here we add the reference count directly instead of invoke - * taosGetDataFromCache, which may return NULL value. - */ - dataBuf->pTableMeta = taosCacheAcquireByData(tscMetaCache, pTableMeta); + //Here we keep the tableMeta to avoid it to be remove by other threads. + dataBuf->pTableMeta = tscTableMetaClone(pTableMeta); assert(initialSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); *dataBlocks = dataBuf; @@ -784,15 +767,15 @@ static int32_t getRowExpandSize(STableMeta* pTableMeta) { return result; } -static void extractTableMeta(SSqlCmd* pCmd) { +static void extractTableNameList(SSqlCmd* pCmd) { pCmd->numOfTables = (int32_t) taosHashGetSize(pCmd->pTableBlockHashList); - pCmd->pTableMetaList = calloc(pCmd->numOfTables, POINTER_BYTES); + pCmd->pTableNameList = calloc(pCmd->numOfTables, POINTER_BYTES); STableDataBlocks **p1 = taosHashIterate(pCmd->pTableBlockHashList, NULL); int32_t i = 0; while(p1) { STableDataBlocks* pBlocks = *p1; - pCmd->pTableMetaList[i++] = taosCacheTransfer(tscMetaCache, (void**) &pBlocks->pTableMeta); + pCmd->pTableNameList[i++] = strndup(pBlocks->tableName, TSDB_TABLE_FNAME_LEN); p1 = taosHashIterate(pCmd->pTableBlockHashList, p1); } @@ -815,7 +798,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql) { STableDataBlocks* dataBuf = NULL; int32_t ret = tscGetDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE, - INSERT_HEAD_SIZE, 0, pOneTableBlock->tableId, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList); + INSERT_HEAD_SIZE, 0, pOneTableBlock->tableName, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList); if (ret != TSDB_CODE_SUCCESS) { tscError("%p failed to prepare the data block buffer for merging table data, code:%d", pSql, ret); taosHashCleanup(pVnodeDataBlockHashList); @@ -849,7 +832,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql) { tscSortRemoveDataBlockDupRows(pOneTableBlock); char* ekey = (char*)pBlocks->data + pOneTableBlock->rowSize*(pBlocks->numOfRows-1); - tscDebug("%p tableId:%s, sid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql, pOneTableBlock->tableId, + tscDebug("%p name:%s, sid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql, pOneTableBlock->tableName, pBlocks->tid, pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(ekey)); int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + expandSize) + sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta); @@ -879,7 +862,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql) { pOneTableBlock = *p; } - extractTableMeta(pCmd); + extractTableNameList(pCmd); // free the table data blocks; pCmd->pDataBlocks = pVnodeDataBlockList; @@ -900,6 +883,7 @@ void tscCloseTscObj(void *param) { rpcClose(pObj->pDnodeConn); pObj->pDnodeConn = NULL; } + tfree(pObj->tscCorMgmtEpSet); pthread_mutex_destroy(&pObj->mutex); @@ -1528,6 +1512,7 @@ int32_t tscTagCondCopy(STagCond* dest, const STagCond* src) { } dest->tbnameCond.uid = src->tbnameCond.uid; + dest->tbnameCond.len = src->tbnameCond.len; memcpy(&dest->joinInfo, &src->joinInfo, sizeof(SJoinInfo)); dest->relType = src->relType; @@ -1823,14 +1808,12 @@ SArray* tscVgroupTableInfoClone(SArray* pVgroupTables) { return pa; } -void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache) { - tscDebug("%p unref %d tables in the tableMeta cache", address, pQueryInfo->numOfTables); - +void clearAllTableMetaInfo(SQueryInfo* pQueryInfo) { for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables); - tscClearTableMetaInfo(pTableMetaInfo, removeFromCache); + tscClearTableMetaInfo(pTableMetaInfo); free(pTableMetaInfo); } @@ -1884,14 +1867,12 @@ STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo* pQueryInfo) { return tscAddTableMetaInfo(pQueryInfo, NULL, NULL, NULL, NULL, NULL); } -void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) { +void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo) { if (pTableMetaInfo == NULL) { return; } - if (pTableMetaInfo->pTableMeta != NULL) { - taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache); - } + tfree(pTableMetaInfo->pTableMeta); pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList); tscColumnListDestroy(pTableMetaInfo->tagColList); @@ -2015,7 +1996,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pNew->pTscObj = pSql->pTscObj; pNew->signature = pNew; - pNew->sqlstr = NULL; + pNew->sqlstr = strdup(pSql->sqlstr); SSqlCmd* pnCmd = &pNew->cmd; memcpy(pnCmd, pCmd, sizeof(SSqlCmd)); @@ -2031,7 +2012,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pnCmd->numOfTables = 0; pnCmd->parseFinished = 1; - pnCmd->pTableMetaList = NULL; + pnCmd->pTableNameList = NULL; pnCmd->pTableBlockHashList = NULL; if (tscAddSubqueryInfo(pnCmd) != TSDB_CODE_SUCCESS) { @@ -2113,8 +2094,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void char* name = pTableMetaInfo->name; STableMetaInfo* pFinalInfo = NULL; - if (pPrevSql == NULL) { // get by name may failed due to the cache cleanup - STableMeta* pTableMeta = taosCacheAcquireByData(tscMetaCache, pTableMetaInfo->pTableMeta); + if (pPrevSql == NULL) { + STableMeta* pTableMeta = tscTableMetaClone(pTableMetaInfo->pTableMeta); assert(pTableMeta != NULL); pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, @@ -2122,15 +2103,15 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void } else { // transfer the ownership of pTableMeta to the newly create sql object. STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0); - STableMeta* pPrevTableMeta = taosCacheTransfer(tscMetaCache, (void**)&pPrevInfo->pTableMeta); - + STableMeta* pPrevTableMeta = tscTableMetaClone(pPrevInfo->pTableMeta); SVgroupsInfo* pVgroupsInfo = pPrevInfo->vgroupList; pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->tagColList, pTableMetaInfo->pVgroupTables); } + // this case cannot be happened if (pFinalInfo->pTableMeta == NULL) { - tscError("%p new subquery failed since no tableMeta in cache, name:%s", pSql, name); + tscError("%p new subquery failed since no tableMeta, name:%s", pSql, name); if (pPrevSql != NULL) { // pass the previous error to client assert(pPrevSql->res.code != TSDB_CODE_SUCCESS); @@ -2557,6 +2538,7 @@ void* tscVgroupInfoClear(SVgroupsInfo *vgroupList) { for(int32_t j = 0; j < pVgroupInfo->numOfEps; ++j) { tfree(pVgroupInfo->epAddr[j].fqdn); } + for(int32_t j = pVgroupInfo->numOfEps; j < TSDB_MAX_REPLICA; j++) { assert( pVgroupInfo->epAddr[j].fqdn == NULL ); } @@ -2610,3 +2592,87 @@ int32_t copyTagData(STagData* dst, const STagData* src) { return 0; } + +STableMeta* createSuperTableMeta(STableMetaMsg* pChild) { + assert(pChild != NULL); + int32_t total = pChild->numOfColumns + pChild->numOfTags; + + STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + sizeof(SSchema) * total); + pTableMeta->tableType = TSDB_SUPER_TABLE; + pTableMeta->tableInfo.numOfTags = pChild->numOfTags; + pTableMeta->tableInfo.numOfColumns = pChild->numOfColumns; + pTableMeta->tableInfo.precision = pChild->precision; + + pTableMeta->id.tid = 0; + pTableMeta->id.uid = pChild->suid; + pTableMeta->tversion = pChild->tversion; + pTableMeta->sversion = pChild->sversion; + + memcpy(pTableMeta->schema, pChild->schema, sizeof(SSchema) * total); + + int32_t num = pTableMeta->tableInfo.numOfColumns; + for(int32_t i = 0; i < num; ++i) { + pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; + } + + return pTableMeta; +} + +uint32_t tscGetTableMetaSize(STableMeta* pTableMeta) { + assert(pTableMeta != NULL); + + int32_t totalCols = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; + return sizeof(STableMeta) + totalCols * sizeof(SSchema); +} + +CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta) { + assert(pTableMeta != NULL); + + CChildTableMeta* cMeta = calloc(1, sizeof(CChildTableMeta)); + cMeta->tableType = TSDB_CHILD_TABLE; + cMeta->vgId = pTableMeta->vgId; + cMeta->id = pTableMeta->id; + tstrncpy(cMeta->sTableName, pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); + + return cMeta; +} + +int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name) { + assert(pChild != NULL); + + uint32_t size = tscGetTableMetaMaxSize(); + STableMeta* p = calloc(1, size); + + taosHashGetClone(tscTableMetaInfo, pChild->sTableName, strnlen(pChild->sTableName, TSDB_TABLE_FNAME_LEN), NULL, p, -1); + if (p->id.uid > 0) { // tableMeta exists, build child table meta and return + pChild->sversion = p->sversion; + pChild->tversion = p->tversion; + + memcpy(&pChild->tableInfo, &p->tableInfo, sizeof(STableInfo)); + int32_t total = pChild->tableInfo.numOfColumns + pChild->tableInfo.numOfTags; + + memcpy(pChild->schema, p->schema, sizeof(SSchema) *total); + + tfree(p); + return TSDB_CODE_SUCCESS; + } else { // super table has been removed, current tableMeta is also expired. remove it here + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); + + tfree(p); + return -1; + } +} + +uint32_t tscGetTableMetaMaxSize() { + return sizeof(STableMeta) + TSDB_MAX_COLUMNS * sizeof(SSchema); +} + +STableMeta* tscTableMetaClone(STableMeta* pTableMeta) { + assert(pTableMeta != NULL); + uint32_t size = tscGetTableMetaSize(pTableMeta); + STableMeta* p = calloc(1, size); + memcpy(p, pTableMeta, size); + return p; +} + + diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 62c06122b6a8ad5d6679f1b477c0da2779dd0539..2dee6dc3bb20c7397ef435d4b7f37e5861f8891e 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -476,6 +476,7 @@ typedef struct { int16_t numOfCols; // the number of columns will be load from vnode SInterval interval; uint16_t tagCondLen; // tag length in current query + uint32_t tbnameCondLen; // table name filter condition string length int16_t numOfGroupCols; // num of group by columns int16_t orderByIdx; int16_t orderType; // used in group by xx order by xxx @@ -494,6 +495,7 @@ typedef struct { int32_t tsNumOfBlocks; // ts comp block numbers int32_t tsOrder; // ts comp block order int32_t numOfTags; // number of tags columns involved + int32_t sqlstrLen; // sql query string SColumnInfo colList[]; } SQueryTableMsg; @@ -725,7 +727,6 @@ typedef struct { typedef struct STableMetaMsg { int32_t contLen; char tableId[TSDB_TABLE_FNAME_LEN]; // table id - char sTableId[TSDB_TABLE_FNAME_LEN]; uint8_t numOfTags; uint8_t precision; uint8_t tableType; @@ -735,6 +736,9 @@ typedef struct STableMetaMsg { int32_t tid; uint64_t uid; SVgroupMsg vgroup; + + char sTableName[TSDB_TABLE_FNAME_LEN]; + uint64_t suid; SSchema schema[]; } STableMetaMsg; diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index db5af5f3a7f6c73755b8184ffcf16d328531dd0f..26e60cdc9cfae17a53e14ad3041ffc058883d1fb 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -2316,11 +2316,12 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { pMeta->precision = pDb->cfg.precision; pMeta->tableType = pTable->info.type; tstrncpy(pMeta->tableId, pTable->info.tableId, TSDB_TABLE_FNAME_LEN); - if (pTable->superTable != NULL) { - tstrncpy(pMeta->sTableId, pTable->superTable->info.tableId, TSDB_TABLE_FNAME_LEN); - } - if (pTable->info.type == TSDB_CHILD_TABLE && pTable->superTable != NULL) { + if (pTable->info.type == TSDB_CHILD_TABLE) { + assert(pTable->superTable != NULL); + tstrncpy(pMeta->sTableName, pTable->superTable->info.tableId, TSDB_TABLE_FNAME_LEN); + + pMeta->suid = pTable->superTable->uid; pMeta->sversion = htons(pTable->superTable->sversion); pMeta->tversion = htons(pTable->superTable->tversion); pMeta->numOfTags = (int8_t)pTable->superTable->numOfTags; diff --git a/src/query/inc/qSyntaxtreefunction.h b/src/query/inc/qArithmeticOperator.h similarity index 91% rename from src/query/inc/qSyntaxtreefunction.h rename to src/query/inc/qArithmeticOperator.h index 6f91d2f7eded38425cf8414b6998e0162557b2c8..f13c63acc3509b13b1696c6360d4135e00079ba2 100644 --- a/src/query/inc/qSyntaxtreefunction.h +++ b/src/query/inc/qArithmeticOperator.h @@ -23,7 +23,7 @@ extern "C" { typedef void (*_bi_consumer_fn_t)(void *left, void *right, int32_t numOfLeft, int32_t numOfRight, void *output, int32_t order); -_bi_consumer_fn_t tGetBiConsumerFn(int32_t leftType, int32_t rightType, int32_t optr); +_bi_consumer_fn_t getArithmeticOperatorFn(int32_t leftType, int32_t rightType, int32_t optr); #ifdef __cplusplus } diff --git a/src/query/inc/qAst.h b/src/query/inc/qAst.h index 28c1c7b838236d10a3f67fa7bbfc15a9b36c4612..39af7261efc222c8a6bcfc809288c256eccb0970 100644 --- a/src/query/inc/qAst.h +++ b/src/query/inc/qAst.h @@ -74,9 +74,7 @@ typedef struct tExprNode { }; } tExprNode; -void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param); - -void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, +void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*cb)(void *, const char*, int32_t)); tExprNode* exprTreeFromBinary(const void* data, size_t size); @@ -87,6 +85,8 @@ void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree); void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)); void tExprTreeDestroy(tExprNode **pExprs, void (*fp)(void*)); +bool exprTreeApplayFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param); + #ifdef __cplusplus } #endif diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 201b3b2abc5df629135cb3daac61b55e2e64cd21..32a68549fabd73104c56ea89a705f7cf5a6a30d1 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -190,7 +190,7 @@ typedef struct SQueryRuntimeEnv { void* pSecQueryHandle; // another thread for bool stableQuery; // super table query or not bool topBotQuery; // TODO used bitwise flag - bool groupbyNormalCol; // denote if this is a groupby normal column query + bool groupbyColumn; // denote if this is a groupby normal column query bool hasTagResults; // if there are tag values in final result or not bool timeWindowInterpo;// if the time window start/end required interpolation bool queryWindowIdentical; // all query time windows are identical for all tables in one group @@ -204,6 +204,8 @@ typedef struct SQueryRuntimeEnv { int32_t* rowCellInfoOffset;// offset value for each row result cell info char** prevRow; char** nextRow; + + SArithmeticSupport *sasArray; } SQueryRuntimeEnv; enum { @@ -237,6 +239,7 @@ typedef struct SQInfo { int32_t dataReady; // denote if query result is ready or not void* rspContext; // response context int64_t startExecTs; // start to exec timestamp + char* sql; // query sql string } SQInfo; #endif // TDENGINE_QUERYEXECUTOR_H diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 38bb0b8a71cbc4d219fd39c209a812660dd0a1b7..51048bbe72368cb6869092b65d5f035f26f3545b 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -112,11 +112,10 @@ extern "C" { #define TOP_BOTTOM_QUERY_LIMIT 100 enum { - MASTER_SCAN = 0x0u, - REVERSE_SCAN = 0x1u, - REPEAT_SCAN = 0x2u, //repeat scan belongs to the master scan - FIRST_STAGE_MERGE = 0x10u, - SECONDARY_STAGE_MERGE = 0x20u, + MASTER_SCAN = 0x0u, + REVERSE_SCAN = 0x1u, + REPEAT_SCAN = 0x2u, //repeat scan belongs to the master scan + MERGE_STAGE = 0x20u, }; #define QUERY_IS_STABLE_QUERY(type) (((type)&TSDB_QUERY_TYPE_STABLE_QUERY) != 0) @@ -191,8 +190,8 @@ typedef struct SQLFunctionCtx { int64_t nStartQueryTimestamp; // timestamp range of current query when function is executed on a specific data block int32_t numOfParams; tVariant param[4]; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param */ - int64_t * ptsList; // corresponding timestamp array list - void * ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ + int64_t *ptsList; // corresponding timestamp array list + void *ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ SQLPreAggVal preAggVals; tVariant tag; @@ -215,18 +214,12 @@ typedef struct SQLAggFuncElem { void (*xFunction)(SQLFunctionCtx *pCtx); // blocks version function void (*xFunctionF)(SQLFunctionCtx *pCtx, int32_t position); // single-row function version - // some sql function require scan data twice or more, e.g.,stddev + // some sql function require scan data twice or more, e.g.,stddev, percentile void (*xNextStep)(SQLFunctionCtx *pCtx); - /* - * finalizer must be called after all xFunction has been executed to - * generated final result. Otherwise, the value in aOutputBuf is a intern result. - */ + // finalizer must be called after all xFunction has been executed to generated final result. void (*xFinalize)(SQLFunctionCtx *pCtx); - - void (*distMergeFunc)(SQLFunctionCtx *pCtx); - - void (*distSecondaryMergeFunc)(SQLFunctionCtx *pCtx); + void (*mergeFunc)(SQLFunctionCtx *pCtx); int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId); } SQLAggFuncElem; diff --git a/src/client/src/tscFunctionImpl.c b/src/query/src/qAggMain.c similarity index 87% rename from src/client/src/tscFunctionImpl.c rename to src/query/src/qAggMain.c index 3a01a9b281d72cba832229576eebcd3e58984a6a..2997d56326b664488a1942df0150343bb5b0f6f9 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/query/src/qAggMain.c @@ -15,27 +15,26 @@ #include "os.h" #include "qAst.h" -#include "qExtbuffer.h" #include "qFill.h" #include "qHistogram.h" #include "qPercentile.h" -#include "qSyntaxtreefunction.h" #include "qTsbuf.h" #include "taosdef.h" #include "taosmsg.h" #include "tscLog.h" #include "tscSubquery.h" -#include "tscompression.h" #include "tsqlfunction.h" -#include "tutil.h" #include "ttype.h" -#define GET_INPUT_CHAR(x) (((char *)((x)->aInputElemBuf)) + ((x)->startOffset) * ((x)->inputBytes)) -#define GET_INPUT_CHAR_INDEX(x, y) (GET_INPUT_CHAR(x) + (y) * (x)->inputBytes) +#define GET_INPUT_DATA_LIST(x) (((char *)((x)->aInputElemBuf)) + ((x)->startOffset) * ((x)->inputBytes)) +#define GET_INPUT_DATA(x, y) (GET_INPUT_DATA_LIST(x) + (y) * (x)->inputBytes) + +#define GET_TS_LIST(x) ((TSKEY*)&((x)->ptsList[(x)->startOffset])) +#define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)]) #define GET_TRUE_DATA_TYPE() \ int32_t type = 0; \ - if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { \ + if (pCtx->currentStage == MERGE_STAGE) { \ type = pCtx->outputType; \ assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); \ } else { \ @@ -62,7 +61,7 @@ } \ aAggs[TSDB_FUNC_TAG].xFunction(__ctx); \ } \ - } while (0); + } while (0) #define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ do { \ @@ -395,7 +394,7 @@ static void count_function(SQLFunctionCtx *pCtx) { } else { if (pCtx->hasNull) { for (int32_t i = 0; i < pCtx->size; ++i) { - char *val = GET_INPUT_CHAR_INDEX(pCtx, i); + char *val = GET_INPUT_DATA(pCtx, i); if (isNull(val, pCtx->inputType)) { continue; } @@ -420,7 +419,7 @@ static void count_function(SQLFunctionCtx *pCtx) { } static void count_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -434,7 +433,7 @@ static void count_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void count_func_merge(SQLFunctionCtx *pCtx) { - int64_t *pData = (int64_t *)GET_INPUT_CHAR(pCtx); + int64_t *pData = (int64_t *)GET_INPUT_DATA_LIST(pCtx); for (int32_t i = 0; i < pCtx->size; ++i) { *((int64_t *)pCtx->aOutputBuf) += pData[i]; } @@ -498,7 +497,7 @@ int32_t no_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId if ((ctx)->hasNull && isNull((char *)&(list)[i], tsdbType)) { \ continue; \ } \ - TSKEY key = (ctx)->ptsList[i]; \ + TSKEY key = GET_TS_DATA(ctx, i); \ UPDATE_DATA(ctx, val, (list)[i], num, sign, key); \ } @@ -525,7 +524,7 @@ static void do_sum(SQLFunctionCtx *pCtx) { *retVal += GET_DOUBLE_VAL((const char*)&(pCtx->preAggVals.statis.sum)); } } else { // computing based on the true data block - void *pData = GET_INPUT_CHAR(pCtx); + void *pData = GET_INPUT_DATA_LIST(pCtx); notNullElems = 0; if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { @@ -558,7 +557,7 @@ static void do_sum(SQLFunctionCtx *pCtx) { } static void do_sum_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -608,21 +607,21 @@ static void sum_function_f(SQLFunctionCtx *pCtx, int32_t index) { } } -static int32_t sum_merge_impl(const SQLFunctionCtx *pCtx) { +static void sum_func_merge(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - + GET_TRUE_DATA_TYPE(); assert(pCtx->stableQuery); - + for (int32_t i = 0; i < pCtx->size; ++i) { - char * input = GET_INPUT_CHAR_INDEX(pCtx, i); + char * input = GET_INPUT_DATA(pCtx, i); SSumInfo *pInput = (SSumInfo *)input; if (pInput->hasResult != DATA_SET_FLAG) { continue; } - + notNullElems++; - + switch (type) { case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: @@ -637,25 +636,7 @@ static int32_t sum_merge_impl(const SQLFunctionCtx *pCtx) { } } } - - return notNullElems; -} - -static void sum_func_merge(SQLFunctionCtx *pCtx) { - int32_t notNullElems = sum_merge_impl(pCtx); - - SET_VAL(pCtx, notNullElems, 1); - SSumInfo *pSumInfo = (SSumInfo *)pCtx->aOutputBuf; - - if (notNullElems > 0) { - // pCtx->numOfIteratedElems += notNullElems; - pSumInfo->hasResult = DATA_SET_FLAG; - } -} -static void sum_func_second_merge(SQLFunctionCtx *pCtx) { - int32_t notNullElems = sum_merge_impl(pCtx); - SET_VAL(pCtx, notNullElems, 1); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); @@ -764,7 +745,7 @@ static void avg_function(SQLFunctionCtx *pCtx) { *pVal += GET_DOUBLE_VAL((const char *)&(pCtx->preAggVals.statis.sum)); } } else { - void *pData = GET_INPUT_CHAR(pCtx); + void *pData = GET_INPUT_DATA_LIST(pCtx); if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { LIST_ADD_N(*pVal, pCtx, pData, int8_t, notNullElems, pCtx->inputType); @@ -799,7 +780,7 @@ static void avg_function(SQLFunctionCtx *pCtx) { } static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -839,33 +820,9 @@ static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void avg_func_merge(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pCtx->stableQuery); - - SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); - char * input = GET_INPUT_CHAR(pCtx); - - for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { - SAvgInfo *pInput = (SAvgInfo *)input; - if (pInput->num == 0) { // current buffer is null - continue; - } - - pAvgInfo->sum += pInput->sum; - pAvgInfo->num += pInput->num; - } - - // if the data set hasResult is not set, the result is null - if (pAvgInfo->num > 0) { - pResInfo->hasResult = DATA_SET_FLAG; - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); - } -} - -static void avg_func_second_merge(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); double *sum = (double*) pCtx->aOutputBuf; - char * input = GET_INPUT_CHAR(pCtx); + char * input = GET_INPUT_DATA_LIST(pCtx); for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { SAvgInfo *pInput = (SAvgInfo *)input; @@ -886,7 +843,7 @@ static void avg_func_second_merge(SQLFunctionCtx *pCtx) { static void avg_finalizer(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { + if (pCtx->currentStage == MERGE_STAGE) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); if (GET_INT64_VAL(GET_ROWCELL_INTERBUF(pResInfo)) <= 0) { @@ -920,6 +877,10 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, if (pCtx->preAggVals.isSet) { *notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull; assert(*notNullElems >= 0); + + if (*notNullElems == 0) { + return; + } if (*notNullElems == 0){ return; @@ -947,7 +908,8 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, if (index < 0 || index >= pCtx->size + pCtx->startOffset) { index = 0; } - + + // the index is the original position, not the relative position key = pCtx->ptsList[index]; } @@ -998,7 +960,9 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, return; } - void *p = GET_INPUT_CHAR(pCtx); + void *p = GET_INPUT_DATA_LIST(pCtx); + TSKEY *tsList = GET_TS_LIST(pCtx); + *notNullElems = 0; if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { @@ -1017,7 +981,7 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, if ((*retVal < pData[i]) ^ isMin) { *retVal = pData[i]; - TSKEY k = pCtx->ptsList[i]; + TSKEY k = tsList[i]; DO_UPDATE_TAG_COLUMNS(pCtx, k); } @@ -1148,7 +1112,7 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp assert(pCtx->stableQuery); for (int32_t i = 0; i < pCtx->size; ++i) { - char *input = GET_INPUT_CHAR_INDEX(pCtx, i); + char *input = GET_INPUT_DATA(pCtx, i); if (input[bytes] != DATA_SET_FLAG) { continue; } @@ -1202,17 +1166,6 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp } static void min_func_merge(SQLFunctionCtx *pCtx) { - int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->inputBytes, pCtx->aOutputBuf, 1); - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { // for super table query, SResultRowCellInfo is not used - char *flag = pCtx->aOutputBuf + pCtx->inputBytes; - *flag = DATA_SET_FLAG; - } -} - -static void min_func_second_merge(SQLFunctionCtx *pCtx) { int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->aOutputBuf, 1); SET_VAL(pCtx, notNullElems, 1); @@ -1224,16 +1177,6 @@ static void min_func_second_merge(SQLFunctionCtx *pCtx) { } static void max_func_merge(SQLFunctionCtx *pCtx) { - int32_t numOfElems = minmax_merge_impl(pCtx, pCtx->inputBytes, pCtx->aOutputBuf, 0); - - SET_VAL(pCtx, numOfElems, 1); - if (numOfElems > 0) { - char *flag = pCtx->aOutputBuf + pCtx->inputBytes; - *flag = DATA_SET_FLAG; - } -} - -static void max_func_second_merge(SQLFunctionCtx *pCtx) { int32_t numOfElem = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->aOutputBuf, 0); SET_VAL(pCtx, numOfElem, 1); @@ -1245,8 +1188,8 @@ static void max_func_second_merge(SQLFunctionCtx *pCtx) { } static void minMax_function_f(SQLFunctionCtx *pCtx, int32_t index, int32_t isMin) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - TSKEY key = pCtx->ptsList[index]; + char *pData = GET_INPUT_DATA(pCtx, index); + TSKEY key = GET_TS_DATA(pCtx, index); int32_t num = 0; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { @@ -1285,7 +1228,7 @@ static void minMax_function_f(SQLFunctionCtx *pCtx, int32_t index, int32_t isMin } static void max_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -1301,7 +1244,7 @@ static void max_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void min_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -1334,7 +1277,7 @@ static void stddev_function(SQLFunctionCtx *pCtx) { double *retVal = &pStd->res; double avg = pStd->avg; - void *pData = GET_INPUT_CHAR(pCtx); + void *pData = GET_INPUT_DATA_LIST(pCtx); switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { @@ -1385,7 +1328,7 @@ static void stddev_function_f(SQLFunctionCtx *pCtx, int32_t index) { avg_function_f(pCtx, index); } else { double avg = pStd->avg; - void * pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void * pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; @@ -1491,14 +1434,14 @@ static void first_function(SQLFunctionCtx *pCtx) { // handle the null value for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } memcpy(pCtx->aOutputBuf, data, pCtx->inputBytes); - TSKEY k = pCtx->ptsList[i]; + TSKEY k = GET_TS_DATA(pCtx, i); DO_UPDATE_TAG_COLUMNS(pCtx, k); SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); @@ -1517,7 +1460,7 @@ static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) { return; } - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -1525,7 +1468,7 @@ static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); - TSKEY ts = pCtx->ptsList[index]; + TSKEY ts = GET_TS_DATA(pCtx, index); DO_UPDATE_TAG_COLUMNS(pCtx, ts); SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); @@ -1534,7 +1477,7 @@ static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void first_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { - int64_t *timestamp = pCtx->ptsList; + int64_t *timestamp = GET_TS_LIST(pCtx); SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); @@ -1565,7 +1508,7 @@ static void first_dist_function(SQLFunctionCtx *pCtx) { // find the first not null value for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } @@ -1583,7 +1526,7 @@ static void first_dist_function(SQLFunctionCtx *pCtx) { } static void first_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -1598,26 +1541,9 @@ static void first_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void first_dist_func_merge(SQLFunctionCtx *pCtx) { - char *pData = GET_INPUT_CHAR(pCtx); - - assert(pCtx->size == 1 && pCtx->stableQuery); - - SFirstLastInfo *pInput = (SFirstLastInfo *)(pData + pCtx->inputBytes); - if (pInput->hasResult != DATA_SET_FLAG) { - return; - } - - SFirstLastInfo *pOutput = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); - if (pOutput->hasResult != DATA_SET_FLAG || pInput->ts < pOutput->ts) { - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes + sizeof(SFirstLastInfo)); - DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); - } -} - -static void first_dist_func_second_merge(SQLFunctionCtx *pCtx) { assert(pCtx->stableQuery); - char * pData = GET_INPUT_CHAR(pCtx); + char * pData = GET_INPUT_DATA_LIST(pCtx); SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes); if (pInput->hasResult != DATA_SET_FLAG) { return; @@ -1652,7 +1578,7 @@ static void last_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { if (!pCtx->requireNull) { continue; @@ -1660,7 +1586,7 @@ static void last_function(SQLFunctionCtx *pCtx) { } memcpy(pCtx->aOutputBuf, data, pCtx->inputBytes); - TSKEY ts = pCtx->ptsList[i]; + TSKEY ts = GET_TS_DATA(pCtx, i); DO_UPDATE_TAG_COLUMNS(pCtx, ts); SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); @@ -1675,7 +1601,7 @@ static void last_function(SQLFunctionCtx *pCtx) { } static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -1689,7 +1615,7 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); - TSKEY ts = pCtx->ptsList[index]; + TSKEY ts = GET_TS_DATA(pCtx, index); DO_UPDATE_TAG_COLUMNS(pCtx, ts); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); @@ -1697,7 +1623,7 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { pResInfo->complete = true; // set query completed } else { // in case of ascending order check, all data needs to be checked SResultRowCellInfo* pResInfo = GET_RES_INFO(pCtx); - TSKEY ts = pCtx->ptsList[index]; + TSKEY ts = GET_TS_DATA(pCtx, index); char* buf = GET_ROWCELL_INTERBUF(pResInfo); if (pResInfo->hasResult != DATA_SET_FLAG || (*(TSKEY*)buf) < ts) { @@ -1711,7 +1637,7 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { - int64_t *timestamp = pCtx->ptsList; + int64_t *timestamp = GET_TS_LIST(pCtx); SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); @@ -1745,7 +1671,7 @@ static void last_dist_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { if (!pCtx->requireNull) { continue; @@ -1769,7 +1695,7 @@ static void last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { return; } - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -1787,32 +1713,13 @@ static void last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); } -static void last_dist_func_merge(SQLFunctionCtx *pCtx) { - char *pData = GET_INPUT_CHAR(pCtx); - - assert(pCtx->size == 1 && pCtx->stableQuery); - - // the input data is null - SFirstLastInfo *pInput = (SFirstLastInfo *)(pData + pCtx->inputBytes); - if (pInput->hasResult != DATA_SET_FLAG) { - return; - } - - SFirstLastInfo *pOutput = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); - if (pOutput->hasResult != DATA_SET_FLAG || pOutput->ts < pInput->ts) { - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes + sizeof(SFirstLastInfo)); - - DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); - } -} - /* * in the secondary merge(local reduce), the output is limited by the * final output size, so the main difference between last_dist_func_merge and second_merge * is: the output data format in computing */ -static void last_dist_func_second_merge(SQLFunctionCtx *pCtx) { - char *pData = GET_INPUT_CHAR(pCtx); +static void last_dist_func_merge(SQLFunctionCtx *pCtx) { + char *pData = GET_INPUT_DATA_LIST(pCtx); SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes); if (pInput->hasResult != DATA_SET_FLAG) { @@ -1841,7 +1748,7 @@ static void last_dist_func_second_merge(SQLFunctionCtx *pCtx) { */ static void last_row_function(SQLFunctionCtx *pCtx) { assert(pCtx->size >= 1); - char *pData = GET_INPUT_CHAR(pCtx); + char *pData = GET_INPUT_DATA_LIST(pCtx); // assign the last element in current data block assignVal(pCtx->aOutputBuf, pData + (pCtx->size - 1) * pCtx->inputBytes, pCtx->inputBytes, pCtx->inputType); @@ -1852,12 +1759,13 @@ static void last_row_function(SQLFunctionCtx *pCtx) { // set the result to final result buffer in case of super table query if (pCtx->stableQuery) { SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); - pInfo1->ts = pCtx->ptsList[pCtx->size - 1]; + pInfo1->ts = GET_TS_DATA(pCtx, pCtx->size - 1); pInfo1->hasResult = DATA_SET_FLAG; DO_UPDATE_TAG_COLUMNS(pCtx, pInfo1->ts); } else { - DO_UPDATE_TAG_COLUMNS(pCtx, pCtx->ptsList[pCtx->size - 1]); + TSKEY ts = GET_TS_DATA(pCtx, pCtx->size - 1); + DO_UPDATE_TAG_COLUMNS(pCtx, ts); } SET_VAL(pCtx, pCtx->size, 1); @@ -1888,7 +1796,7 @@ static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int6 dst->timestamp = tsKey; int32_t size = 0; - if (stage == SECONDARY_STAGE_MERGE || stage == FIRST_STAGE_MERGE) { + if (stage == MERGE_STAGE) { memcpy(dst->pTags, pTags, (size_t)pTagInfo->tagsLen); } else { // the tags are dumped from the ctx tag fields for (int32_t i = 0; i < pTagInfo->numOfTagCols; ++i) { @@ -2151,7 +2059,7 @@ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); // only the first_stage_merge is directly written data into final output buffer - if (pCtx->stableQuery && pCtx->currentStage != SECONDARY_STAGE_MERGE) { + if (pCtx->stableQuery && pCtx->currentStage != MERGE_STAGE) { return (STopBotInfo*) pCtx->aOutputBuf; } else { // during normal table query and super table at the secondary_stage, result is written to intermediate buffer return GET_ROWCELL_INTERBUF(pResInfo); @@ -2249,13 +2157,15 @@ static void top_function(SQLFunctionCtx *pCtx) { assert(pRes->num >= 0); for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); + TSKEY ts = GET_TS_DATA(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } notNullElems++; - do_top_function_add(pRes, (int32_t)pCtx->param[0].i64Key, data, pCtx->ptsList[i], pCtx->inputType, &pCtx->tagInfo, NULL, 0); + do_top_function_add(pRes, (int32_t)pCtx->param[0].i64Key, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); } if (!pCtx->hasNull) { @@ -2272,7 +2182,7 @@ static void top_function(SQLFunctionCtx *pCtx) { } static void top_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -2281,36 +2191,16 @@ static void top_function_f(SQLFunctionCtx *pCtx, int32_t index) { assert(pRes->num >= 0); SET_VAL(pCtx, 1, 1); - do_top_function_add(pRes, (int32_t)pCtx->param[0].i64Key, pData, pCtx->ptsList[index], pCtx->inputType, &pCtx->tagInfo, NULL, - 0); + TSKEY ts = GET_TS_DATA(pCtx, index); + + do_top_function_add(pRes, (int32_t)pCtx->param[0].i64Key, pData, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } static void top_func_merge(SQLFunctionCtx *pCtx) { - char *input = GET_INPUT_CHAR(pCtx); - - STopBotInfo *pInput = (STopBotInfo *)input; - if (pInput->num <= 0) { - return; - } - - // remmap the input buffer may cause the struct pointer invalid, so rebuild the STopBotInfo is necessary - buildTopBotStruct(pInput, pCtx); - - assert(pCtx->stableQuery && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1); - - STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - - for (int32_t i = 0; i < pInput->num; ++i) { - do_top_function_add(pOutput, (int32_t)pCtx->param[0].i64Key, &pInput->res[i]->v.i64Key, pInput->res[i]->timestamp, - pCtx->inputType, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); - } -} - -static void top_func_second_merge(SQLFunctionCtx *pCtx) { - STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_CHAR(pCtx); + STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx); // construct the input data struct from binary data buildTopBotStruct(pInput, pCtx); @@ -2338,14 +2228,15 @@ static void bottom_function(SQLFunctionCtx *pCtx) { STopBotInfo *pRes = getTopBotOutputInfo(pCtx); for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); + TSKEY ts = GET_TS_DATA(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } notNullElems++; - do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64Key, data, pCtx->ptsList[i], pCtx->inputType, &pCtx->tagInfo, NULL, - 0); + do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64Key, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); } if (!pCtx->hasNull) { @@ -2362,43 +2253,23 @@ static void bottom_function(SQLFunctionCtx *pCtx) { } static void bottom_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); + TSKEY ts = GET_TS_DATA(pCtx, index); + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } STopBotInfo *pRes = getTopBotOutputInfo(pCtx); SET_VAL(pCtx, 1, 1); - do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64Key, pData, pCtx->ptsList[index], pCtx->inputType, &pCtx->tagInfo, - NULL, 0); + do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64Key, pData, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } static void bottom_func_merge(SQLFunctionCtx *pCtx) { - char *input = GET_INPUT_CHAR(pCtx); - - STopBotInfo *pInput = (STopBotInfo *)input; - if (pInput->num <= 0) { - return; - } - - // remmap the input buffer may cause the struct pointer invalid, so rebuild the STopBotInfo is necessary - buildTopBotStruct(pInput, pCtx); - - assert(pCtx->stableQuery && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1); - - STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - - for (int32_t i = 0; i < pInput->num; ++i) { - do_bottom_function_add(pOutput, (int32_t)pCtx->param[0].i64Key, &pInput->res[i]->v.i64Key, pInput->res[i]->timestamp, - pCtx->inputType, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); - } -} - -static void bottom_func_second_merge(SQLFunctionCtx *pCtx) { - STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_CHAR(pCtx); + STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx); // construct the input data struct from binary data buildTopBotStruct(pInput, pCtx); @@ -2494,7 +2365,7 @@ static void percentile_function(SQLFunctionCtx *pCtx) { pInfo->numOfElems += (pCtx->size - pCtx->preAggVals.statis.numOfNull); } else { for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } @@ -2519,7 +2390,7 @@ static void percentile_function(SQLFunctionCtx *pCtx) { // the second stage, calculate the true percentile value for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } @@ -2533,7 +2404,7 @@ static void percentile_function(SQLFunctionCtx *pCtx) { } static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -2610,7 +2481,7 @@ static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo* pInfo = NULL; - if (pCtx->stableQuery && pCtx->currentStage != SECONDARY_STAGE_MERGE) { + if (pCtx->stableQuery && pCtx->currentStage != MERGE_STAGE) { pInfo = (SAPercentileInfo*) pCtx->aOutputBuf; } else { pInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -2641,7 +2512,7 @@ static void apercentile_function(SQLFunctionCtx *pCtx) { assert(pInfo->pHisto->elems != NULL); for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } @@ -2665,7 +2536,7 @@ static void apercentile_function(SQLFunctionCtx *pCtx) { } static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -2683,42 +2554,7 @@ static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void apercentile_func_merge(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pCtx->stableQuery); - - SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_CHAR(pCtx); - - pInput->pHisto = (SHistogramInfo*) ((char *)pInput + sizeof(SAPercentileInfo)); - pInput->pHisto->elems = (SHistBin*) ((char *)pInput->pHisto + sizeof(SHistogramInfo)); - - if (pInput->pHisto->numOfElems <= 0) { - return; - } - - size_t size = sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1); - - SAPercentileInfo *pOutput = getAPerctInfo(pCtx); //(SAPercentileInfo *)pCtx->aOutputBuf; - SHistogramInfo * pHisto = pOutput->pHisto; - - if (pHisto->numOfElems <= 0) { - memcpy(pHisto, pInput->pHisto, size); - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - } else { - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - - SHistogramInfo *pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN); - memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN); - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - - tHistogramDestroy(&pRes); - } - - SET_VAL(pCtx, 1, 1); - pResInfo->hasResult = DATA_SET_FLAG; -} - -static void apercentile_func_second_merge(SQLFunctionCtx *pCtx) { - SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_CHAR(pCtx); + SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_DATA_LIST(pCtx); pInput->pHisto = (SHistogramInfo*) ((char *)pInput + sizeof(SAPercentileInfo)); pInput->pHisto->elems = (SHistBin*) ((char *)pInput->pHisto + sizeof(SHistogramInfo)); @@ -2753,7 +2589,7 @@ static void apercentile_finalizer(SQLFunctionCtx *pCtx) { SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo *pOutput = GET_ROWCELL_INTERBUF(pResInfo); - if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { + if (pCtx->currentStage == MERGE_STAGE) { if (pResInfo->hasResult == DATA_SET_FLAG) { // check for null assert(pOutput->pHisto->numOfElems > 0); @@ -2821,7 +2657,7 @@ static void leastsquares_function(SQLFunctionCtx *pCtx) { double(*param)[3] = pInfo->mat; double x = pInfo->startVal; - void *pData = GET_INPUT_CHAR(pCtx); + void *pData = GET_INPUT_DATA_LIST(pCtx); int32_t numOfElem = 0; switch (pCtx->inputType) { @@ -2881,7 +2717,7 @@ static void leastsquares_function(SQLFunctionCtx *pCtx) { } static void leastsquares_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -2983,7 +2819,7 @@ static void col_project_function(SQLFunctionCtx *pCtx) { INC_INIT_VAL(pCtx, pCtx->size); - char *pData = GET_INPUT_CHAR(pCtx); + char *pData = GET_INPUT_DATA_LIST(pCtx); if (pCtx->order == TSDB_ORDER_ASC) { memcpy(pCtx->aOutputBuf, pData, (size_t) pCtx->size * pCtx->inputBytes); } else { @@ -3008,7 +2844,7 @@ static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { } INC_INIT_VAL(pCtx, 1); - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); pCtx->aOutputBuf += pCtx->inputBytes; @@ -3062,7 +2898,7 @@ static void tag_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void copy_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, pCtx->size, 1); - char *pData = GET_INPUT_CHAR(pCtx); + char *pData = GET_INPUT_DATA_LIST(pCtx); assignVal(pCtx->aOutputBuf, pData, pCtx->inputBytes, pCtx->inputType); } @@ -3082,7 +2918,7 @@ static bool diff_function_setup(SQLFunctionCtx *pCtx) { // TODO difference in date column static void diff_function(SQLFunctionCtx *pCtx) { - void *data = GET_INPUT_CHAR(pCtx); + void *data = GET_INPUT_DATA_LIST(pCtx); bool isFirstBlock = (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED); int32_t notNullElems = 0; @@ -3090,8 +2926,9 @@ static void diff_function(SQLFunctionCtx *pCtx) { int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1; - TSKEY * pTimestamp = pCtx->ptsOutputBuf; - + TSKEY* pTimestamp = pCtx->ptsOutputBuf; + TSKEY* tsList = GET_TS_LIST(pCtx); + switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { int32_t *pData = (int32_t *)data; @@ -3107,13 +2944,13 @@ static void diff_function(SQLFunctionCtx *pCtx) { pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = (int32_t)(pData[i] - pCtx->param[1].i64Key); - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } else { *pOutput = (int32_t)(pData[i] - pCtx->param[1].i64Key); // direct previous may be null - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; @@ -3139,13 +2976,13 @@ static void diff_function(SQLFunctionCtx *pCtx) { pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = pData[i] - pCtx->param[1].i64Key; - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } else { *pOutput = pData[i] - pCtx->param[1].i64Key; - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; @@ -3171,12 +3008,12 @@ static void diff_function(SQLFunctionCtx *pCtx) { pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = pData[i] - pCtx->param[1].dKey; - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } else { *pOutput = pData[i] - pCtx->param[1].dKey; - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } @@ -3201,13 +3038,13 @@ static void diff_function(SQLFunctionCtx *pCtx) { pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = (float)(pData[i] - pCtx->param[1].dKey); - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } else { *pOutput = (float)(pData[i] - pCtx->param[1].dKey); - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; @@ -3234,12 +3071,12 @@ static void diff_function(SQLFunctionCtx *pCtx) { pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = (int16_t)(pData[i] - pCtx->param[1].i64Key); - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } else { *pOutput = (int16_t)(pData[i] - pCtx->param[1].i64Key); - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; @@ -3265,13 +3102,13 @@ static void diff_function(SQLFunctionCtx *pCtx) { pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = (int8_t)(pData[i] - pCtx->param[1].i64Key); - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } else { *pOutput = (int8_t)(pData[i] - pCtx->param[1].i64Key); - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; @@ -3312,12 +3149,12 @@ static void diff_function(SQLFunctionCtx *pCtx) { } else { \ *(type *)(ctx)->aOutputBuf = *(type *)(d) - (*(type *)(&(ctx)->param[1].i64Key)); \ *(type *)(&(ctx)->param[1].i64Key) = *(type *)(d); \ - *(int64_t *)(ctx)->ptsOutputBuf = (ctx)->ptsList[index]; \ + *(int64_t *)(ctx)->ptsOutputBuf = GET_TS_DATA(ctx, index); \ } \ } while (0); static void diff_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -3337,7 +3174,7 @@ static void diff_function_f(SQLFunctionCtx *pCtx, int32_t index) { } else { *(int32_t *)pCtx->aOutputBuf = *(int32_t *)pData - (int32_t)pCtx->param[1].i64Key; pCtx->param[1].i64Key = *(int32_t *)pData; - *(int64_t *)pCtx->ptsOutputBuf = pCtx->ptsList[index]; + *(int64_t *)pCtx->ptsOutputBuf = GET_TS_DATA(pCtx, index); } break; }; @@ -3390,7 +3227,7 @@ static void arithmetic_function(SQLFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->numOfRes += pCtx->size; SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; - tExprTreeCalcTraverse(sas->pArithExpr->pExpr, pCtx->size, pCtx->aOutputBuf, sas, pCtx->order, getArithColumnData); + arithmeticTreeTraverse(sas->pArithExpr->pExpr, pCtx->size, pCtx->aOutputBuf, sas, pCtx->order, getArithColumnData); pCtx->aOutputBuf += pCtx->outputBytes * pCtx->size; pCtx->param[1].pz = NULL; @@ -3401,7 +3238,7 @@ static void arithmetic_function_f(SQLFunctionCtx *pCtx, int32_t index) { SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; sas->offset = index; - tExprTreeCalcTraverse(sas->pArithExpr->pExpr, 1, pCtx->aOutputBuf, sas, pCtx->order, getArithColumnData); + arithmeticTreeTraverse(sas->pArithExpr->pExpr, 1, pCtx->aOutputBuf, sas, pCtx->order, getArithColumnData); pCtx->aOutputBuf += pCtx->outputBytes; } @@ -3432,7 +3269,7 @@ static bool spread_function_setup(SQLFunctionCtx *pCtx) { SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); // this is the server-side setup function in client-side, the secondary merge do not need this procedure - if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { + if (pCtx->currentStage == MERGE_STAGE) { pCtx->param[0].dKey = DBL_MAX; pCtx->param[3].dKey = -DBL_MAX; } else { @@ -3481,7 +3318,7 @@ static void spread_function(SQLFunctionCtx *pCtx) { goto _spread_over; } - void *pData = GET_INPUT_CHAR(pCtx); + void *pData = GET_INPUT_DATA_LIST(pCtx); numOfElems = 0; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { @@ -3517,7 +3354,7 @@ static void spread_function(SQLFunctionCtx *pCtx) { } static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -3559,45 +3396,12 @@ static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) { } } -void spread_func_merge(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pCtx->stableQuery); - - SSpreadInfo *pResData = GET_ROWCELL_INTERBUF(pResInfo); - - int32_t notNullElems = 0; - for (int32_t i = 0; i < pCtx->size; ++i) { - SSpreadInfo *input = (SSpreadInfo *)GET_INPUT_CHAR_INDEX(pCtx, i); - - /* no assign tag, the value is null */ - if (input->hasResult != DATA_SET_FLAG) { - continue; - } - - if (pResData->min > input->min) { - pResData->min = input->min; - } - - if (pResData->max < input->max) { - pResData->max = input->max; - } - - pResData->hasResult = DATA_SET_FLAG; - notNullElems++; - } - - if (notNullElems > 0) { - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); - pResInfo->hasResult = DATA_SET_FLAG; - } -} - /* * here we set the result value back to the intermediate buffer, to apply the finalize the function * the final result is generated in spread_function_finalizer */ void spread_func_sec_merge(SQLFunctionCtx *pCtx) { - SSpreadInfo *pData = (SSpreadInfo *)GET_INPUT_CHAR(pCtx); + SSpreadInfo *pData = (SSpreadInfo *)GET_INPUT_DATA_LIST(pCtx); if (pData->hasResult != DATA_SET_FLAG) { return; } @@ -3620,7 +3424,7 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { */ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { + if (pCtx->currentStage == MERGE_STAGE) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); if (pResInfo->hasResult != DATA_SET_FLAG) { @@ -3675,26 +3479,25 @@ static double twa_get_area(SPoint1 s, SPoint1 e) { return val; } -static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t index, int32_t size) { +static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t size) { int32_t notNullElems = 0; - TSKEY *primaryKey = pCtx->ptsList; - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); + TSKEY *tsList = GET_TS_LIST(pCtx); int32_t i = index; int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); SPoint1* last = &pInfo->p; if (pCtx->start.key != INT64_MIN) { - assert((pCtx->start.key < primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_ASC) || - (pCtx->start.key > primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_DESC)); + assert((pCtx->start.key < tsList[i] && pCtx->order == TSDB_ORDER_ASC) || + (pCtx->start.key > tsList[i] && pCtx->order == TSDB_ORDER_DESC)); assert(last->key == INT64_MIN); - last->key = primaryKey[tsIndex + i]; - GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); + last->key = tsList[i]; + GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_DATA(pCtx, index)); pInfo->dOutput += twa_get_area(pCtx->start, *last); @@ -3703,8 +3506,8 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t notNullElems++; i += step; } else if (pInfo->p.key == INT64_MIN) { - last->key = primaryKey[tsIndex + i]; - GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); + last->key = tsList[i]; + GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_DATA(pCtx, index)); pInfo->hasResult = DATA_SET_FLAG; pInfo->win.skey = last->key; @@ -3715,78 +3518,78 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t // calculate the value of switch(pCtx->inputType) { case TSDB_DATA_TYPE_TINYINT: { - int8_t *val = (int8_t*) GET_INPUT_CHAR_INDEX(pCtx, 0); + int8_t *val = (int8_t*) GET_INPUT_DATA(pCtx, 0); for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - SPoint1 st = {.key = primaryKey[i + tsIndex], .val = val[i]}; + SPoint1 st = {.key = tsList[i], .val = val[i]}; pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } break; } case TSDB_DATA_TYPE_SMALLINT: { - int16_t *val = (int16_t*) GET_INPUT_CHAR_INDEX(pCtx, 0); + int16_t *val = (int16_t*) GET_INPUT_DATA(pCtx, 0); for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - SPoint1 st = {.key = primaryKey[i + tsIndex], .val = val[i]}; + SPoint1 st = {.key = tsList[i], .val = val[i]}; pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } break; } case TSDB_DATA_TYPE_INT: { - int32_t *val = (int32_t*) GET_INPUT_CHAR_INDEX(pCtx, 0); + int32_t *val = (int32_t*) GET_INPUT_DATA(pCtx, 0); for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - SPoint1 st = {.key = primaryKey[i + tsIndex], .val = val[i]}; + SPoint1 st = {.key = tsList[i], .val = val[i]}; pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } break; } case TSDB_DATA_TYPE_BIGINT: { - int64_t *val = (int64_t*) GET_INPUT_CHAR_INDEX(pCtx, 0); + int64_t *val = (int64_t*) GET_INPUT_DATA(pCtx, 0); for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - SPoint1 st = {.key = primaryKey[i + tsIndex], .val = (double) val[i]}; + SPoint1 st = {.key = tsList[i], .val = (double) val[i]}; pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } break; } case TSDB_DATA_TYPE_FLOAT: { - float *val = (float*) GET_INPUT_CHAR_INDEX(pCtx, 0); + float *val = (float*) GET_INPUT_DATA(pCtx, 0); for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - SPoint1 st = {.key = primaryKey[i + tsIndex], .val = val[i]}; + SPoint1 st = {.key = tsList[i], .val = val[i]}; pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } break; } case TSDB_DATA_TYPE_DOUBLE: { - double *val = (double*) GET_INPUT_CHAR_INDEX(pCtx, 0); + double *val = (double*) GET_INPUT_DATA(pCtx, 0); for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - SPoint1 st = {.key = primaryKey[i + tsIndex], .val = val[i]}; + SPoint1 st = {.key = tsList[i], .val = val[i]}; pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -3806,7 +3609,7 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t } static void twa_function(SQLFunctionCtx *pCtx) { - void *data = GET_INPUT_CHAR(pCtx); + void *data = GET_INPUT_DATA_LIST(pCtx); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -3814,11 +3617,15 @@ static void twa_function(SQLFunctionCtx *pCtx) { // skip null value int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); int32_t i = (pCtx->order == TSDB_ORDER_ASC)? 0:(pCtx->size - 1); - while (pCtx->hasNull && i < pCtx->size && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) { + while (pCtx->hasNull && i < pCtx->size && i >= 0 && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) { i += step; } - int32_t notNullElems = twa_function_impl(pCtx, pCtx->startOffset, i, pCtx->size); + int32_t notNullElems = 0; + if (i >= 0 && i < pCtx->size) { + notNullElems = twa_function_impl(pCtx, i, pCtx->size); + } + SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { @@ -3830,135 +3637,15 @@ static void twa_function(SQLFunctionCtx *pCtx) { } } -//TODO refactor static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - int32_t notNullElems = 0; - TSKEY *primaryKey = pCtx->ptsList; - + int32_t notNullElems = twa_function_impl(pCtx, index, 1); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - int32_t i = pCtx->startOffset; - int32_t size = pCtx->size; - - if (pCtx->start.key != INT64_MIN) { - assert(pInfo->p.key == INT64_MIN); - - pInfo->p.key = primaryKey[index]; - GET_TYPED_DATA(pInfo->p.val, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); - - pInfo->dOutput += twa_get_area(pCtx->start, pInfo->p); - - pInfo->hasResult = DATA_SET_FLAG; - pInfo->win.skey = pCtx->start.key; - notNullElems++; - i += 1; - } else if (pInfo->p.key == INT64_MIN) { - pInfo->p.key = primaryKey[index]; - GET_TYPED_DATA(pInfo->p.val, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); - - pInfo->hasResult = DATA_SET_FLAG; - pInfo->win.skey = pInfo->p.key; - notNullElems++; - i += 1; - } - - // calculate the value of - switch(pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: { - int8_t *val = (int8_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - - SPoint1 st = {.key = primaryKey[i + index], .val = val[i]}; - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *val = (int16_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - - SPoint1 st = {.key = primaryKey[i + index], .val = val[i]}; - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_INT: { - int32_t *val = (int32_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - - SPoint1 st = {.key = primaryKey[i + index], .val = val[i]}; - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_BIGINT: { - int64_t *val = (int64_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - - SPoint1 st = {.key = primaryKey[i + index], .val = (double) val[i]}; - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float *val = (float*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - - SPoint1 st = {.key = primaryKey[i + index], .val = val[i]}; - pInfo->dOutput += twa_get_area(pInfo->p, st);//((val[i] + pInfo->p.val) / 2) * (primaryKey[i + index] - pInfo->p.key); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *val = (double*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - - SPoint1 st = {.key = primaryKey[i + index], .val = val[i]}; - pInfo->dOutput += twa_get_area(pInfo->p, st);//((val[i] + pInfo->p.val) / 2) * (primaryKey[i + index] - pInfo->p.key); - pInfo->p = st; - } - break; - } - default: assert(0); - } - - // the last interpolated time window value - if (pCtx->end.key != INT64_MIN) { - pInfo->dOutput += twa_get_area(pInfo->p, pCtx->end);//((pInfo->p.val + pCtx->end.val) / 2) * (pCtx->end.key - pInfo->p.key); - pInfo->p = pCtx->end; - } - - pInfo->win.ekey = pInfo->p.key; - SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { @@ -3970,34 +3657,6 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { } } -static void twa_func_merge(SQLFunctionCtx *pCtx) { - assert(pCtx->stableQuery); - - STwaInfo *pBuf = (STwaInfo *)pCtx->aOutputBuf; - char * indicator = pCtx->aInputElemBuf; - - int32_t numOfNotNull = 0; - for (int32_t i = 0; i < pCtx->size; ++i, indicator += sizeof(STwaInfo)) { - STwaInfo *pInput = (STwaInfo*) indicator; - - if (pInput->hasResult != DATA_SET_FLAG) { - continue; - } - - numOfNotNull++; - pBuf->dOutput += pInput->dOutput; - - pBuf->win = pInput->win; - pBuf->p = pInput->p; - } - - SET_VAL(pCtx, numOfNotNull, 1); - - if (numOfNotNull > 0) { - pBuf->hasResult = DATA_SET_FLAG; - } -} - /* * To copy the input to interResBuf to avoid the input buffer space be over writen * by next input data. The TWA function only applies to each table, so no merge procedure @@ -4040,8 +3699,10 @@ static void interp_function(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SInterpInfoDetail* pInfo = GET_ROWCELL_INTERBUF(pResInfo); + assert(pCtx->startOffset == 0); + if (pCtx->size == 1) { - char *pData = GET_INPUT_CHAR(pCtx); + char *pData = GET_INPUT_DATA_LIST(pCtx); assignVal(pCtx->aOutputBuf, pData, pCtx->inputBytes, pCtx->inputType); } else { /* @@ -4067,13 +3728,13 @@ static void interp_function(SQLFunctionCtx *pCtx) { } else if (pInfo->type == TSDB_FILL_SET_VALUE) { tVariantDump(&pCtx->param[1], pCtx->aOutputBuf, pCtx->inputType, true); } else if (pInfo->type == TSDB_FILL_PREV) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, 0); + char *data = GET_INPUT_DATA(pCtx, 0); assignVal(pCtx->aOutputBuf, data, pCtx->outputBytes, pCtx->outputType); SET_VAL(pCtx, pCtx->size, 1); } else if (pInfo->type == TSDB_FILL_LINEAR) { - char *data1 = GET_INPUT_CHAR_INDEX(pCtx, 0); - char *data2 = GET_INPUT_CHAR_INDEX(pCtx, 1); + char *data1 = GET_INPUT_DATA(pCtx, 0); + char *data2 = GET_INPUT_DATA(pCtx, 1); TSKEY key1 = pCtx->ptsList[0]; TSKEY key2 = pCtx->ptsList[1]; @@ -4135,14 +3796,14 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STSBuf * pTSbuf = ((STSCompInfo *)(GET_ROWCELL_INTERBUF(pResInfo)))->pTSBuf; - const char *input = GET_INPUT_CHAR(pCtx); + const char *input = GET_INPUT_DATA_LIST(pCtx); // primary ts must be existed, so no need to check its existance if (pCtx->order == TSDB_ORDER_ASC) { tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64Key, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE); } else { for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *d = GET_INPUT_CHAR_INDEX(pCtx, i); + char *d = GET_INPUT_DATA(pCtx, i); tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64Key, &pCtx->tag, d, (int32_t)TSDB_KEYSIZE); } } @@ -4152,7 +3813,7 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) { } static void ts_comp_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -4214,7 +3875,6 @@ static double do_calc_rate(const SRateInfo* pRateInfo) { return resultVal; } - static bool rate_function_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { return false; @@ -4236,18 +3896,17 @@ static bool rate_function_setup(SQLFunctionCtx *pCtx) { return true; } - static void rate_function(SQLFunctionCtx *pCtx) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - int32_t notNullElems = 0; - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = pCtx->ptsList; + int32_t notNullElems = 0; + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); + TSKEY *primaryKey = GET_TS_LIST(pCtx); tscDebug("%p rate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); for (int32_t i = 0; i < pCtx->size; ++i) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, i); + char *pData = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { tscDebug("%p rate_function() index of null data:%d", pCtx, i); continue; @@ -4295,7 +3954,7 @@ static void rate_function(SQLFunctionCtx *pCtx) { } static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -4303,8 +3962,8 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { // NOTE: keep the intermediate result into the interResultBuf SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = pCtx->ptsList; - + TSKEY *primaryKey = GET_TS_LIST(pCtx); + int64_t v = 0; GET_TYPED_DATA(v, int64_t, pCtx->inputType, pData); @@ -4336,42 +3995,6 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { } } - - -static void rate_func_merge(SQLFunctionCtx *pCtx) { - assert(pCtx->stableQuery); - - tscDebug("rate_func_merge() size:%d", pCtx->size); - - SRateInfo *pBuf = (SRateInfo *)pCtx->aOutputBuf; - char *indicator = pCtx->aInputElemBuf; - - assert(1 == pCtx->size); - - int32_t numOfNotNull = 0; - for (int32_t i = 0; i < pCtx->size; ++i, indicator += sizeof(SRateInfo)) { - SRateInfo *pInput = (SRateInfo *)indicator; - if (DATA_SET_FLAG != pInput->hasResult) { - continue; - } - - numOfNotNull++; - memcpy(pBuf, pInput, sizeof(SRateInfo)); - tscDebug("%p rate_func_merge() isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64, - pCtx, pInput->isIRate, pInput->firstKey, pInput->lastKey, pInput->firstValue, pInput->lastValue, pInput->CorrectionValue); - } - - SET_VAL(pCtx, numOfNotNull, 1); - - if (numOfNotNull > 0) { - pBuf->hasResult = DATA_SET_FLAG; - } - - return; -} - - - static void rate_func_copy(SQLFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); @@ -4380,12 +4003,10 @@ static void rate_func_copy(SQLFunctionCtx *pCtx) { pResInfo->hasResult = ((SRateInfo*)pCtx->aInputElemBuf)->hasResult; SRateInfo* pRateInfo = (SRateInfo*)pCtx->aInputElemBuf; - tscDebug("%p rate_func_second_merge() firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " hasResult:%d", + tscDebug("%p rate_func_merge() firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " hasResult:%d", pCtx, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, pRateInfo->hasResult); } - - static void rate_finalizer(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); @@ -4409,14 +4030,13 @@ static void rate_finalizer(SQLFunctionCtx *pCtx) { doFinalizer(pCtx); } - static void irate_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = pCtx->ptsList; - + TSKEY *primaryKey = GET_TS_LIST(pCtx); + tscDebug("%p irate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); if (pCtx->size < 1) { @@ -4424,7 +4044,7 @@ static void irate_function(SQLFunctionCtx *pCtx) { } for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, i); + char *pData = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { tscDebug("%p irate_function() index of null data:%d", pCtx, i); continue; @@ -4467,15 +4087,15 @@ static void irate_function(SQLFunctionCtx *pCtx) { } static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } // NOTE: keep the intermediate result into the interResultBuf - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = pCtx->ptsList; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); + TSKEY *primaryKey = GET_TS_LIST(pCtx); int64_t v = 0; GET_TYPED_DATA(v, int64_t, pCtx->inputType, pData); @@ -4505,7 +4125,7 @@ static void do_sumrate_merge(SQLFunctionCtx *pCtx) { assert(pCtx->stableQuery); SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - char * input = GET_INPUT_CHAR(pCtx); + char * input = GET_INPUT_DATA_LIST(pCtx); for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { SRateInfo *pInput = (SRateInfo *)input; @@ -4537,11 +4157,6 @@ static void sumrate_func_merge(SQLFunctionCtx *pCtx) { do_sumrate_merge(pCtx); } -static void sumrate_func_second_merge(SQLFunctionCtx *pCtx) { - tscDebug("%p sumrate_func_second_merge() process ...", pCtx); - do_sumrate_merge(pCtx); -} - static void sumrate_finalizer(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); @@ -4605,7 +4220,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, count_func_merge, - count_func_merge, count_load_data_info, }, { @@ -4620,7 +4234,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, sum_func_merge, - sum_func_second_merge, statisRequired, }, { @@ -4635,7 +4248,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, avg_finalizer, avg_func_merge, - avg_func_second_merge, statisRequired, }, { @@ -4650,7 +4262,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, min_func_merge, - min_func_second_merge, statisRequired, }, { @@ -4665,7 +4276,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, max_func_merge, - max_func_second_merge, statisRequired, }, { @@ -4680,7 +4290,6 @@ SQLAggFuncElem aAggs[] = {{ stddev_next_step, stddev_finalizer, noop1, - noop1, dataBlockRequired, }, { @@ -4695,7 +4304,6 @@ SQLAggFuncElem aAggs[] = {{ percentile_next_step, percentile_finalizer, noop1, - noop1, dataBlockRequired, }, { @@ -4710,7 +4318,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, apercentile_finalizer, apercentile_func_merge, - apercentile_func_second_merge, dataBlockRequired, }, { @@ -4725,7 +4332,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, noop1, - noop1, firstFuncRequired, }, { @@ -4740,7 +4346,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, noop1, - noop1, lastFuncRequired, }, { @@ -4755,8 +4360,7 @@ SQLAggFuncElem aAggs[] = {{ noop2, no_next_step, last_row_finalizer, - noop1, - last_dist_func_second_merge, + last_dist_func_merge, dataBlockRequired, }, { @@ -4772,7 +4376,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, top_bottom_func_finalizer, top_func_merge, - top_func_second_merge, dataBlockRequired, }, { @@ -4788,7 +4391,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, top_bottom_func_finalizer, bottom_func_merge, - bottom_func_second_merge, dataBlockRequired, }, { @@ -4802,7 +4404,6 @@ SQLAggFuncElem aAggs[] = {{ spread_function_f, no_next_step, spread_function_finalizer, - spread_func_merge, spread_func_sec_merge, count_load_data_info, }, @@ -4817,7 +4418,6 @@ SQLAggFuncElem aAggs[] = {{ twa_function_f, no_next_step, twa_function_finalizer, - twa_func_merge, twa_function_copy, dataBlockRequired, }, @@ -4833,7 +4433,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, leastsquares_finalizer, noop1, - noop1, dataBlockRequired, }, { @@ -4848,7 +4447,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, no_data_info, }, { @@ -4863,7 +4461,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, dataBlockRequired, }, { @@ -4878,7 +4475,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, no_data_info, }, { @@ -4893,7 +4489,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, ts_comp_finalize, copy_function, - copy_function, dataBlockRequired, }, { @@ -4908,7 +4503,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, no_data_info, }, { @@ -4923,7 +4517,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, dataBlockRequired, }, { @@ -4938,7 +4531,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, no_data_info, }, { @@ -4953,7 +4545,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, dataBlockRequired, }, { @@ -4968,7 +4559,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, noop1, - noop1, dataBlockRequired, }, // distributed version used in two-stage aggregation processes @@ -4984,7 +4574,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, first_dist_func_merge, - first_dist_func_second_merge, firstDistFuncRequired, }, { @@ -4999,7 +4588,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, last_dist_func_merge, - last_dist_func_second_merge, lastDistFuncRequired, }, { @@ -5013,7 +4601,6 @@ SQLAggFuncElem aAggs[] = {{ do_sum_f, // todo filter handle no_next_step, doFinalizer, - noop1, copy_function, dataBlockRequired, }, @@ -5028,7 +4615,6 @@ SQLAggFuncElem aAggs[] = {{ rate_function_f, no_next_step, rate_finalizer, - rate_func_merge, rate_func_copy, dataBlockRequired, }, @@ -5043,7 +4629,6 @@ SQLAggFuncElem aAggs[] = {{ irate_function_f, no_next_step, rate_finalizer, - rate_func_merge, rate_func_copy, dataBlockRequired, }, @@ -5059,7 +4644,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, sumrate_finalizer, sumrate_func_merge, - sumrate_func_second_merge, dataBlockRequired, }, { @@ -5074,7 +4658,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, sumrate_finalizer, sumrate_func_merge, - sumrate_func_second_merge, dataBlockRequired, }, { @@ -5089,7 +4672,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, sumrate_finalizer, sumrate_func_merge, - sumrate_func_second_merge, dataBlockRequired, }, { @@ -5104,7 +4686,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, sumrate_finalizer, sumrate_func_merge, - sumrate_func_second_merge, dataBlockRequired, }, { @@ -5119,6 +4700,5 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, noop1, noop1, - noop1, dataBlockRequired, }}; diff --git a/src/query/src/qSyntaxtreefunction.c b/src/query/src/qArithmeticOperator.c similarity index 99% rename from src/query/src/qSyntaxtreefunction.c rename to src/query/src/qArithmeticOperator.c index 7f7fca2c1e0ca6ce85f0df1c1b6e6dce82d8c000..0955d48df0bacbdf748c8a06f6a6364387d9be9d 100644 --- a/src/query/src/qSyntaxtreefunction.c +++ b/src/query/src/qArithmeticOperator.c @@ -15,7 +15,7 @@ #include "os.h" -#include "qSyntaxtreefunction.h" +#include "qArithmeticOperator.h" #include "taosdef.h" #include "tutil.h" @@ -1234,7 +1234,7 @@ _bi_consumer_fn_t rem_function_arraylist[8][10] = { //////////////////////////////////////////////////////////////////////////////////////////////////////////// -_bi_consumer_fn_t tGetBiConsumerFn(int32_t leftType, int32_t rightType, int32_t optr) { +_bi_consumer_fn_t getArithmeticOperatorFn(int32_t leftType, int32_t rightType, int32_t optr) { switch (optr) { case TSDB_BINARY_OP_ADD: return add_function_arraylist[leftType][rightType]; diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index a65f4a6dc95a586896372a648dc19ad2bb88b154..c23a79419660f0e474854e005db86a18971c0c96 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -16,29 +16,18 @@ #include "os.h" #include "exception.h" +#include "qArithmeticOperator.h" #include "qAst.h" -#include "qSyntaxtreefunction.h" #include "taosdef.h" #include "taosmsg.h" #include "tarray.h" #include "tbuffer.h" #include "tcompare.h" #include "tname.h" +#include "tschemautil.h" #include "tsdb.h" #include "tskiplist.h" #include "tsqlfunction.h" -#include "tstoken.h" -#include "tschemautil.h" - -typedef struct { - char* v; - int32_t optr; -} SEndPoint; - -typedef struct { - SEndPoint* start; - SEndPoint* end; -} SQueryCond; static uint8_t UNUSED_FUNC isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight) { if (pLeft->nodeType == TSQL_NODE_COL) { @@ -53,323 +42,6 @@ static uint8_t UNUSED_FUNC isQueryOnPrimaryKey(const char *primaryColumnName, co } } -void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)) { - if (pNode == NULL) { - return; - } - - if (pNode->nodeType == TSQL_NODE_EXPR) { - tExprTreeDestroy(&pNode, fp); - } else if (pNode->nodeType == TSQL_NODE_VALUE) { - tVariantDestroy(pNode->pVal); - } else if (pNode->nodeType == TSQL_NODE_COL) { - free(pNode->pSchema); - } - - free(pNode); -} - -void tExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { - if (*pExpr == NULL) { - return; - } - - if ((*pExpr)->nodeType == TSQL_NODE_EXPR) { - tExprTreeDestroy(&(*pExpr)->_node.pLeft, fp); - tExprTreeDestroy(&(*pExpr)->_node.pRight, fp); - - if (fp != NULL) { - fp((*pExpr)->_node.info); - } - } else if ((*pExpr)->nodeType == TSQL_NODE_VALUE) { - tVariantDestroy((*pExpr)->pVal); - free((*pExpr)->pVal); - } else if ((*pExpr)->nodeType == TSQL_NODE_COL) { - free((*pExpr)->pSchema); - } - - free(*pExpr); - *pExpr = NULL; -} - -// todo check for malloc failure -static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) { - int32_t optr = queryColInfo->optr; - - if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL || - optr == TSDB_RELATION_EQUAL || optr == TSDB_RELATION_NOT_EQUAL) { - pCond->start = calloc(1, sizeof(SEndPoint)); - pCond->start->optr = queryColInfo->optr; - pCond->start->v = queryColInfo->q; - } else if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { - pCond->end = calloc(1, sizeof(SEndPoint)); - pCond->end->optr = queryColInfo->optr; - pCond->end->v = queryColInfo->q; - } else if (optr == TSDB_RELATION_IN || optr == TSDB_RELATION_LIKE) { - assert(0); - } - - return TSDB_CODE_SUCCESS; -} - -static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* result) { - SSkipListIterator* iter = NULL; - - SQueryCond cond = {0}; - if (setQueryCond(pQueryInfo, &cond) != TSDB_CODE_SUCCESS) { - //todo handle error - } - - if (cond.start != NULL) { - iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_ASC); - } else { - iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->type, TSDB_ORDER_DESC); - } - - if (cond.start != NULL) { - int32_t optr = cond.start->optr; - - if (optr == TSDB_RELATION_EQUAL) { // equals - while(tSkipListIterNext(iter)) { - SSkipListNode* pNode = tSkipListIterGet(iter); - - int32_t ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v); - if (ret != 0) { - break; - } - - STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(result, &info); - } - } else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) { // greater equal - bool comp = true; - int32_t ret = 0; - - while(tSkipListIterNext(iter)) { - SSkipListNode* pNode = tSkipListIterGet(iter); - - if (comp) { - ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v); - assert(ret >= 0); - } - - if (ret == 0 && optr == TSDB_RELATION_GREATER) { - continue; - } else { - STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(result, &info); - comp = false; - } - } - } else if (optr == TSDB_RELATION_NOT_EQUAL) { // not equal - bool comp = true; - - while(tSkipListIterNext(iter)) { - SSkipListNode* pNode = tSkipListIterGet(iter); - comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0); - if (comp) { - continue; - } - - STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(result, &info); - } - - tSkipListDestroyIter(iter); - - comp = true; - iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_DESC); - while(tSkipListIterNext(iter)) { - SSkipListNode* pNode = tSkipListIterGet(iter); - comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0); - if (comp) { - continue; - } - - STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(result, &info); - } - - } else { - assert(0); - } - } else { - int32_t optr = cond.end ? cond.end->optr : TSDB_RELATION_INVALID; - if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { - bool comp = true; - int32_t ret = 0; - - while (tSkipListIterNext(iter)) { - SSkipListNode *pNode = tSkipListIterGet(iter); - - if (comp) { - ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.end->v); - assert(ret <= 0); - } - - if (ret == 0 && optr == TSDB_RELATION_LESS) { - continue; - } else { - STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(result, &info); - comp = false; // no need to compare anymore - } - } - } else { - assert(pQueryInfo->optr == TSDB_RELATION_ISNULL || pQueryInfo->optr == TSDB_RELATION_NOTNULL); - - while (tSkipListIterNext(iter)) { - SSkipListNode *pNode = tSkipListIterGet(iter); - - bool isnull = isNull(SL_GET_NODE_KEY(pSkipList, pNode), pQueryInfo->sch.type); - if ((pQueryInfo->optr == TSDB_RELATION_ISNULL && isnull) || - (pQueryInfo->optr == TSDB_RELATION_NOTNULL && (!isnull))) { - STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(result, &info); - } - } - } - } - - free(cond.start); - free(cond.end); - tSkipListDestroyIter(iter); -} - -static bool filterItem(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param) { - tExprNode *pLeft = pExpr->_node.pLeft; - tExprNode *pRight = pExpr->_node.pRight; - - //non-leaf nodes, recursively traverse the expression tree in the post-root order - if (pLeft->nodeType == TSQL_NODE_EXPR && pRight->nodeType == TSQL_NODE_EXPR) { - if (pExpr->_node.optr == TSDB_RELATION_OR) { // or - if (filterItem(pLeft, pItem, param)) { - return true; - } - - // left child does not satisfy the query condition, try right child - return filterItem(pRight, pItem, param); - } else { // and - if (!filterItem(pLeft, pItem, param)) { - return false; - } - - return filterItem(pRight, pItem, param); - } - } - - // handle the leaf node - param->setupInfoFn(pExpr, param->pExtInfo); - return param->nodeFilterFn(pItem, pExpr->_node.info); -} - -static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSkipList *pSkipList, SExprTraverseSupp *param ) { - SSkipListIterator* iter = tSkipListCreateIter(pSkipList); - - while (tSkipListIterNext(iter)) { - SSkipListNode *pNode = tSkipListIterGet(iter); - if (filterItem(pExpr, pNode, param)) { - taosArrayPush(pResult, &(SL_GET_NODE_DATA(pNode))); - } - } - tSkipListDestroyIter(iter); -} - -static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* res, __result_filter_fn_t filterFp) { - SSkipListIterator* iter = tSkipListCreateIter(pSkipList); - - while (tSkipListIterNext(iter)) { - bool addToResult = false; - - SSkipListNode *pNode = tSkipListIterGet(iter); - char * pData = SL_GET_NODE_DATA(pNode); - - tstr *name = (tstr*) tsdbGetTableName((void*) pData); - - // todo speed up by using hash - if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { - if (pQueryInfo->optr == TSDB_RELATION_IN) { - addToResult = pQueryInfo->compare(name, pQueryInfo->q); - } else if (pQueryInfo->optr == TSDB_RELATION_LIKE) { - addToResult = !pQueryInfo->compare(name, pQueryInfo->q); - } - } else { - addToResult = filterFp(pNode, pQueryInfo); - } - - if (addToResult) { - STableKeyInfo info = {.pTable = (void*)pData, .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(res, &info); - } - } - - tSkipListDestroyIter(iter); -} - -// post-root order traverse syntax tree -void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param) { - if (pExpr == NULL) { - return; - } - - tExprNode *pLeft = pExpr->_node.pLeft; - tExprNode *pRight = pExpr->_node.pRight; - - // column project - if (pLeft->nodeType != TSQL_NODE_EXPR && pRight->nodeType != TSQL_NODE_EXPR) { - assert(pLeft->nodeType == TSQL_NODE_COL && (pRight->nodeType == TSQL_NODE_VALUE || pRight->nodeType == TSQL_NODE_DUMMY)); - - param->setupInfoFn(pExpr, param->pExtInfo); - - tQueryInfo *pQueryInfo = pExpr->_node.info; - if (pQueryInfo->indexed && pQueryInfo->optr != TSDB_RELATION_LIKE) { - tQueryIndexColumn(pSkipList, pQueryInfo, result); - } else { - tQueryIndexlessColumn(pSkipList, pQueryInfo, result, param->nodeFilterFn); - } - - return; - } - - // The value of hasPK is always 0. - uint8_t weight = pLeft->_node.hasPK + pRight->_node.hasPK; - assert(weight == 0 && pSkipList != NULL && taosArrayGetSize(result) == 0); - - //apply the hierarchical expression to every node in skiplist for find the qualified nodes - tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param); - -#if 0 - /* - * (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here - * - * first, we filter results based on the skiplist index, which is the initial filter stage, - * then, we conduct the secondary filter operation based on the result from the initial filter stage. - */ - assert(pExpr->_node.optr == TSDB_RELATION_AND); - - tExprNode *pFirst = NULL; - tExprNode *pSecond = NULL; - if (pLeft->_node.hasPK == 1) { - pFirst = pLeft; - pSecond = pRight; - } else { - pFirst = pRight; - pSecond = pLeft; - } - - assert(pFirst != pSecond && pFirst != NULL && pSecond != NULL); - - // we filter the result based on the skiplist index in the first place - tExprTreeTraverse(pFirst, pSkipList, result, param); - - /* - * recursively perform the filter operation based on the initial results, - * So, we do not set the skip list index as a parameter - */ - tExprTreeTraverse(pSecond, NULL, result, param); -#endif -} - static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOfRows) { switch(type) { case TSDB_DATA_TYPE_TINYINT: { @@ -430,7 +102,73 @@ static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOf } } -void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, +void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)) { + if (pNode == NULL) { + return; + } + + if (pNode->nodeType == TSQL_NODE_EXPR) { + tExprTreeDestroy(&pNode, fp); + } else if (pNode->nodeType == TSQL_NODE_VALUE) { + tVariantDestroy(pNode->pVal); + } else if (pNode->nodeType == TSQL_NODE_COL) { + free(pNode->pSchema); + } + + free(pNode); +} + +void tExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { + if (*pExpr == NULL) { + return; + } + + if ((*pExpr)->nodeType == TSQL_NODE_EXPR) { + tExprTreeDestroy(&(*pExpr)->_node.pLeft, fp); + tExprTreeDestroy(&(*pExpr)->_node.pRight, fp); + + if (fp != NULL) { + fp((*pExpr)->_node.info); + } + } else if ((*pExpr)->nodeType == TSQL_NODE_VALUE) { + tVariantDestroy((*pExpr)->pVal); + free((*pExpr)->pVal); + } else if ((*pExpr)->nodeType == TSQL_NODE_COL) { + free((*pExpr)->pSchema); + } + + free(*pExpr); + *pExpr = NULL; +} + +bool exprTreeApplayFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param) { + tExprNode *pLeft = pExpr->_node.pLeft; + tExprNode *pRight = pExpr->_node.pRight; + + //non-leaf nodes, recursively traverse the expression tree in the post-root order + if (pLeft->nodeType == TSQL_NODE_EXPR && pRight->nodeType == TSQL_NODE_EXPR) { + if (pExpr->_node.optr == TSDB_RELATION_OR) { // or + if (exprTreeApplayFilter(pLeft, pItem, param)) { + return true; + } + + // left child does not satisfy the query condition, try right child + return exprTreeApplayFilter(pRight, pItem, param); + } else { // and + if (!exprTreeApplayFilter(pLeft, pItem, param)) { + return false; + } + + return exprTreeApplayFilter(pRight, pItem, param); + } + } + + // handle the leaf node + param->setupInfoFn(pExpr, param->pExtInfo); + return param->nodeFilterFn(pItem, pExpr->_node.info); +} + +void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*getSourceDataBlock)(void *, const char*, int32_t)) { if (pExprs == NULL) { return; @@ -442,7 +180,7 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, /* the left output has result from the left child syntax tree */ char *pLeftOutput = (char*)malloc(sizeof(int64_t) * numOfRows); if (pLeft->nodeType == TSQL_NODE_EXPR) { - tExprTreeCalcTraverse(pLeft, numOfRows, pLeftOutput, param, order, getSourceDataBlock); + arithmeticTreeTraverse(pLeft, numOfRows, pLeftOutput, param, order, getSourceDataBlock); } /* the right output has result from the right child syntax tree */ @@ -450,7 +188,7 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, char *pdata = malloc(sizeof(int64_t) * numOfRows); if (pRight->nodeType == TSQL_NODE_EXPR) { - tExprTreeCalcTraverse(pRight, numOfRows, pRightOutput, param, order, getSourceDataBlock); + arithmeticTreeTraverse(pRight, numOfRows, pRightOutput, param, order, getSourceDataBlock); } if (pLeft->nodeType == TSQL_NODE_EXPR) { @@ -459,11 +197,11 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, * exprLeft + exprRight * the type of returned value of one expression is always double float precious */ - _bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); fp(pLeftOutput, pRightOutput, numOfRows, numOfRows, pOutput, TSDB_ORDER_ASC); } else if (pRight->nodeType == TSQL_NODE_COL) { // exprLeft + columnRight - _bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, pRight->pSchema->type, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(TSDB_DATA_TYPE_DOUBLE, pRight->pSchema->type, pExprs->_node.optr); // set input buffer char *pInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId); @@ -475,14 +213,14 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, } } else if (pRight->nodeType == TSQL_NODE_VALUE) { // exprLeft + 12 - _bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, pRight->pVal->nType, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(TSDB_DATA_TYPE_DOUBLE, pRight->pVal->nType, pExprs->_node.optr); fp(pLeftOutput, &pRight->pVal->i64Key, numOfRows, 1, pOutput, TSDB_ORDER_ASC); } } else if (pLeft->nodeType == TSQL_NODE_COL) { // column data specified on left-hand-side char *pLeftInputData = getSourceDataBlock(param, pLeft->pSchema->name, pLeft->pSchema->colId); if (pRight->nodeType == TSQL_NODE_EXPR) { // columnLeft + expr2 - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(pLeft->pSchema->type, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); if (order == TSDB_ORDER_DESC) { reverseCopy(pdata, pLeftInputData, pLeft->pSchema->type, numOfRows); @@ -494,12 +232,12 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, } else if (pRight->nodeType == TSQL_NODE_COL) { // columnLeft + columnRight // column data specified on right-hand-side char *pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId); - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, pRight->pSchema->type, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(pLeft->pSchema->type, pRight->pSchema->type, pExprs->_node.optr); // both columns are descending order, do not reverse the source data fp(pLeftInputData, pRightInputData, numOfRows, numOfRows, pOutput, order); } else if (pRight->nodeType == TSQL_NODE_VALUE) { // columnLeft + 12 - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, pRight->pVal->nType, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(pLeft->pSchema->type, pRight->pVal->nType, pExprs->_node.optr); if (order == TSDB_ORDER_DESC) { reverseCopy(pdata, pLeftInputData, pLeft->pSchema->type, numOfRows); @@ -511,13 +249,13 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, } else { // column data specified on left-hand-side if (pRight->nodeType == TSQL_NODE_EXPR) { // 12 + expr2 - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(pLeft->pVal->nType, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); fp(&pLeft->pVal->i64Key, pRightOutput, 1, numOfRows, pOutput, TSDB_ORDER_ASC); } else if (pRight->nodeType == TSQL_NODE_COL) { // 12 + columnRight // column data specified on right-hand-side char *pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId); - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, pRight->pSchema->type, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(pLeft->pVal->nType, pRight->pSchema->type, pExprs->_node.optr); if (order == TSDB_ORDER_DESC) { reverseCopy(pdata, pRightInputData, pRight->pSchema->type, numOfRows); @@ -527,7 +265,7 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, } } else if (pRight->nodeType == TSQL_NODE_VALUE) { // 12 + 12 - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, pRight->pVal->nType, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(pLeft->pVal->nType, pRight->pVal->nType, pExprs->_node.optr); fp(&pLeft->pVal->i64Key, &pRight->pVal->i64Key, 1, 1, pOutput, TSDB_ORDER_ASC); } } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 1f76d479687c987af7be02e7680630e43ce7e147..c257e8468f98839f4ac70d5227558ba9b01af0c6 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -86,7 +86,6 @@ typedef struct { int32_t status; // query status TSKEY lastKey; // the lastKey value before query executed STimeWindow w; // whole query time window - STimeWindow curWindow; // current query window int32_t windowIndex; // index of active time window result for interval query STSCursor cur; } SQueryStatusInfo; @@ -718,7 +717,7 @@ static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_se return forwardStep; } -static void doUpdateResultRowIndex(SResultRowInfo*pResultRowInfo, TSKEY lastKey, bool ascQuery) { +static void doUpdateResultRowIndex(SResultRowInfo*pResultRowInfo, TSKEY lastKey, bool ascQuery, bool timeWindowInterpo) { int64_t skey = TSKEY_INITIAL_VAL; int32_t i = 0; for (i = pResultRowInfo->size - 1; i >= 0; --i) { @@ -728,10 +727,22 @@ static void doUpdateResultRowIndex(SResultRowInfo*pResultRowInfo, TSKEY lastKey, } // new closed result rows - if ((pResult->win.ekey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery)) { - closeResultRow(pResultRowInfo, i); + if (timeWindowInterpo) { + if (pResult->endInterp && ((pResult->win.skey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery))) { + if (i > 0) { // the first time window, the startInterp is false. + assert(pResult->startInterp); + } + + closeResultRow(pResultRowInfo, i); + } else { + skey = pResult->win.skey; + } } else { - skey = pResult->win.skey; + if ((pResult->win.ekey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery)) { + closeResultRow(pResultRowInfo, i); + } else { + skey = pResult->win.skey; + } } } @@ -752,13 +763,13 @@ static void doUpdateResultRowIndex(SResultRowInfo*pResultRowInfo, TSKEY lastKey, } } -static void updateResultRowIndex(SResultRowInfo* pResultRowInfo, STableQueryInfo* pTableQueryInfo, bool ascQuery) { +static void updateResultRowIndex(SResultRowInfo* pResultRowInfo, STableQueryInfo* pTableQueryInfo, bool ascQuery, bool timeWindowInterpo) { if ((pTableQueryInfo->lastKey > pTableQueryInfo->win.ekey && ascQuery) || (pTableQueryInfo->lastKey < pTableQueryInfo->win.ekey && (!ascQuery))) { closeAllResultRows(pResultRowInfo); pResultRowInfo->curIndex = pResultRowInfo->size - 1; } else { int32_t step = ascQuery? 1:-1; - doUpdateResultRowIndex(pResultRowInfo, pTableQueryInfo->lastKey - step, ascQuery); + doUpdateResultRowIndex(pResultRowInfo, pTableQueryInfo->lastKey - step, ascQuery, timeWindowInterpo); } } @@ -814,9 +825,6 @@ static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1); int32_t functionId = pQuery->pExpr1[k].base.functionId; - if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { - pCtx[k].ptsList = &tsCol[pCtx[k].startOffset]; - } // not a whole block involved in query processing, statistics data can not be used // NOTE: the original value of isSet have been changed here @@ -954,8 +962,7 @@ static void* getDataBlockImpl(SArray* pDataBlock, int32_t colId) { return NULL; } -static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int32_t col, int32_t size, - SArray *pDataBlock) { +static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int32_t col, int32_t size, SArray *pDataBlock) { if (pDataBlock == NULL) { return NULL; } @@ -966,15 +973,9 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas int32_t functionId = pQuery->pExpr1[col].base.functionId; if (functionId == TSDB_FUNC_ARITHM) { sas->pArithExpr = &pQuery->pExpr1[col]; - - sas->offset = (QUERY_IS_ASC_QUERY(pQuery))? pQuery->pos : pQuery->pos - (size - 1); - sas->colList = pQuery->colList; - sas->numOfCols = pQuery->numOfCols; - sas->data = calloc(pQuery->numOfCols, POINTER_BYTES); - - if (sas->data == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } + sas->offset = (QUERY_IS_ASC_QUERY(pQuery))? pQuery->pos : pQuery->pos - (size - 1); + sas->colList = pQuery->colList; + sas->numOfCols = pQuery->numOfCols; // here the pQuery->colList and sas->colList are identical int32_t numOfCols = (int32_t)taosArrayGetSize(pDataBlock); @@ -1077,13 +1078,13 @@ static bool setTimeWindowInterpolationEndTs(SQueryRuntimeEnv* pRuntimeEnv, int32 return true; } -static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray* pDataBlock) { +static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray* pDataBlock, + int32_t rowIndex) { if (pDataBlock == NULL) { return; } SQuery* pQuery = pRuntimeEnv->pQuery; - int32_t rowIndex = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->rows-1:0; for (int32_t k = 0; k < pQuery->numOfCols; ++k) { SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k); memcpy(pRuntimeEnv->prevRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes); @@ -1166,15 +1167,10 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * tsCols = (TSKEY *)(pColInfo->pData); } - SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport)); - if (sasArray == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - SQInfo *pQInfo = GET_QINFO_ADDR(pRuntimeEnv); for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &sasArray[k], k, pQInfo->vgId); + char *dataBlock = getDataBlock(pRuntimeEnv, &pRuntimeEnv->sasArray[k], k, pDataBlockInfo->rows, pDataBlock); + setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &pRuntimeEnv->sasArray[k], k, pQInfo->vgId); } int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); @@ -1266,25 +1262,12 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * _end: if (pRuntimeEnv->timeWindowInterpo) { - saveDataBlockLastRow(pRuntimeEnv, pDataBlockInfo, pDataBlock); - } - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - if (pQuery->pExpr1[i].base.functionId != TSDB_FUNC_ARITHM) { - continue; - } - - tfree(sasArray[i].data); + int32_t rowIndex = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->rows-1:0; + saveDataBlockLastRow(pRuntimeEnv, pDataBlockInfo, pDataBlock, rowIndex); } - - tfree(sasArray); } static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) { - if (isNull(pData, type)) { // ignore the null value - return -1; - } - SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; // not assign result buffer yet, add new result buffer, TODO remove it @@ -1296,8 +1279,7 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat } else if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); qError("QInfo:%p group by not supported on double/float columns, abort", pQInfo); - - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); + return -1; } SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true, groupIndex); @@ -1513,7 +1495,7 @@ static void setTimeWindowEKeyInterp(SQueryRuntimeEnv* pRuntimeEnv, SArray* pData } static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, - SResultRowInfo *pWindowResInfo, SArray *pDataBlock) { + SResultRowInfo *pWindowResInfo, SArray *pDataBlock) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); @@ -1525,12 +1507,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS SColumnInfoData* pColumnInfoData = (SColumnInfoData *)taosArrayGet(pDataBlock, 0); TSKEY *tsCols = (pColumnInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP)? (TSKEY*) pColumnInfoData->pData:NULL; - bool groupbyColumnValue = pRuntimeEnv->groupbyNormalCol; - - SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport)); - if (sasArray == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } + bool groupbyColumnValue = pRuntimeEnv->groupbyColumn; int16_t type = 0; int16_t bytes = 0; @@ -1542,8 +1519,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &sasArray[k], k, pQInfo->vgId); + char *dataBlock = getDataBlock(pRuntimeEnv, &pRuntimeEnv->sasArray[k], k, pDataBlockInfo->rows, pDataBlock); + setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &pRuntimeEnv->sasArray[k], k, pQInfo->vgId); pCtx[k].size = 1; } @@ -1588,7 +1565,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS // interval window query, decide the time window according to the primary timestamp if (QUERY_IS_INTERVAL_QUERY(pQuery)) { int32_t prevWindowIndex = curTimeWindowIndex(pWindowResInfo); - int64_t ts = tsCols[offset]; + int64_t ts = tsCols[offset]; STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); @@ -1628,10 +1605,9 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } doRowwiseApplyFunctions(pRuntimeEnv, &win, offset); + int32_t index = pWindowResInfo->curIndex; STimeWindow nextWin = win; - int32_t index = pWindowResInfo->curIndex; - while (1) { getNextTimeWindow(pQuery, &nextWin); if ((nextWin.skey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || @@ -1653,15 +1629,19 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS doRowwiseApplyFunctions(pRuntimeEnv, &nextWin, offset); } + // restore the index, add the result row will move the index pWindowResInfo->curIndex = index; } else { // other queries // decide which group this rows belongs to according to current state value if (groupbyColumnValue) { char *val = groupbyColumnData + bytes * offset; + if (isNull(val, type)) { // ignore the null value + continue; + } int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, val, type, bytes, item->groupIndex); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - continue; + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } } @@ -1686,27 +1666,20 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } _end: - assert(offset >= 0); - if (tsCols != NULL) { - item->lastKey = tsCols[offset] + step; - } else { - item->lastKey = (QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey) + step; + assert(offset >= 0 && tsCols != NULL); + if (prevTs != INT64_MIN) { + assert(prevRowIndex >= 0); + item->lastKey = prevTs + step; } - if (pRuntimeEnv->pTsBuf != NULL) { - item->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); + // In case of all rows in current block are not qualified + if (pRuntimeEnv->timeWindowInterpo && prevRowIndex != -1) { + saveDataBlockLastRow(pRuntimeEnv, pDataBlockInfo, pDataBlock, prevRowIndex); } - // todo refactor: extract method - for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { - if (pQuery->pExpr1[i].base.functionId != TSDB_FUNC_ARITHM) { - continue; - } - - tfree(sasArray[i].data); + if (pRuntimeEnv->pTsBuf != NULL) { + item->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); } - - free(sasArray); } static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, @@ -1716,7 +1689,7 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl STableQueryInfo* pTableQueryInfo = pQuery->current; SResultRowInfo* pResultRowInfo = &pRuntimeEnv->windowResInfo; - if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->groupbyNormalCol) { + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->groupbyColumn) { rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, pDataBlock); } else { blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, searchFn, pDataBlock); @@ -1728,9 +1701,9 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl // interval query with limit applied int32_t numOfRes = 0; - if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) { + if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyColumn) { numOfRes = pResultRowInfo->size; - updateResultRowIndex(pResultRowInfo, pTableQueryInfo, QUERY_IS_ASC_QUERY(pQuery)); + updateResultRowIndex(pResultRowInfo, pTableQueryInfo, QUERY_IS_ASC_QUERY(pQuery), pRuntimeEnv->timeWindowInterpo); } else { // projection query numOfRes = (int32_t) getNumOfResult(pRuntimeEnv); @@ -1786,7 +1759,7 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY uint32_t status = aAggs[functionId].nStatus; if (((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) && (tsCol != NULL)) { - pCtx->ptsList = &tsCol[pCtx->startOffset]; + pCtx->ptsList = tsCol; } if (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) { @@ -1904,8 +1877,9 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutput, sizeof(SQLFunctionCtx)); pRuntimeEnv->offset = calloc(pQuery->numOfOutput, sizeof(int16_t)); pRuntimeEnv->rowCellInfoOffset = calloc(pQuery->numOfOutput, sizeof(int32_t)); + pRuntimeEnv->sasArray = calloc(pQuery->numOfOutput, sizeof(SArithmeticSupport)); - if (pRuntimeEnv->offset == NULL || pRuntimeEnv->pCtx == NULL || pRuntimeEnv->rowCellInfoOffset == NULL) { + if (pRuntimeEnv->offset == NULL || pRuntimeEnv->pCtx == NULL || pRuntimeEnv->rowCellInfoOffset == NULL || pRuntimeEnv->sasArray == NULL) { goto _clean; } @@ -1981,18 +1955,24 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order pCtx->param[1].i64Key = pQuery->order.orderColId; } + if (functionId == TSDB_FUNC_ARITHM) { + pRuntimeEnv->sasArray[i].data = calloc(pQuery->numOfCols, POINTER_BYTES); + if (pRuntimeEnv->sasArray[i].data == NULL) { + goto _clean; + } + } + if (i > 0) { pRuntimeEnv->offset[i] = pRuntimeEnv->offset[i - 1] + pRuntimeEnv->pCtx[i - 1].outputBytes; pRuntimeEnv->rowCellInfoOffset[i] = pRuntimeEnv->rowCellInfoOffset[i - 1] + sizeof(SResultRowCellInfo) + pQuery->pExpr1[i - 1].interBytes; } - } *(int64_t*) pRuntimeEnv->prevRow[0] = INT64_MIN; // if it is group by normal column, do not set output buffer, the output buffer is pResult // fixed output query/multi-output query for normal table - if (!pRuntimeEnv->groupbyNormalCol && !pRuntimeEnv->stableQuery && !QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQuery)) { + if (!pRuntimeEnv->groupbyColumn && !pRuntimeEnv->stableQuery && !QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQuery)) { resetDefaultResInfoOutputBuf(pRuntimeEnv); } @@ -2007,6 +1987,7 @@ _clean: tfree(pRuntimeEnv->pCtx); tfree(pRuntimeEnv->offset); tfree(pRuntimeEnv->rowCellInfoOffset); + tfree(pRuntimeEnv->sasArray); return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -2050,6 +2031,14 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { tfree(pRuntimeEnv->pCtx); } + if (pRuntimeEnv->sasArray != NULL) { + for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { + tfree(pRuntimeEnv->sasArray[i].data); + } + + tfree(pRuntimeEnv->sasArray); + } + pRuntimeEnv->pFillInfo = taosDestroyFillInfo(pRuntimeEnv->pFillInfo); destroyResultBuf(pRuntimeEnv->pResultBuf); @@ -2102,7 +2091,7 @@ static bool isFixedOutputQuery(SQueryRuntimeEnv* pRuntimeEnv) { } // Note:top/bottom query is fixed output query - if (pRuntimeEnv->topBotQuery || pRuntimeEnv->groupbyNormalCol) { + if (pRuntimeEnv->topBotQuery || pRuntimeEnv->groupbyColumn) { return true; } @@ -2743,7 +2732,7 @@ static void ensureOutputBufferSimple(SQueryRuntimeEnv* pRuntimeEnv, int32_t capa static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pBlockInfo) { // in case of prj/diff query, ensure the output buffer is sufficient to accommodate the results of current block SQuery* pQuery = pRuntimeEnv->pQuery; - if (!QUERY_IS_INTERVAL_QUERY(pQuery) && !pRuntimeEnv->groupbyNormalCol && !isFixedOutputQuery(pRuntimeEnv) && !isTSCompQuery(pQuery)) { + if (!QUERY_IS_INTERVAL_QUERY(pQuery) && !pRuntimeEnv->groupbyColumn && !isFixedOutputQuery(pRuntimeEnv) && !isTSCompQuery(pQuery)) { SResultRec *pRec = &pQuery->rec; if (pQuery->rec.capacity - pQuery->rec.rows < pBlockInfo->rows) { @@ -2967,50 +2956,6 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { } } -static UNUSED_FUNC void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SResultRow *pWindowRes, bool mergeFlag) { - SQuery * pQuery = pRuntimeEnv->pQuery; - SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId); - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionId = pQuery->pExpr1[i].base.functionId; - if (!mergeFlag) { - pCtx[i].aOutputBuf = pCtx[i].aOutputBuf + pCtx[i].outputBytes; - pCtx[i].currentStage = FIRST_STAGE_MERGE; - - RESET_RESULT_INFO(pCtx[i].resultInfo); - aAggs[functionId].init(&pCtx[i]); - } - - pCtx[i].hasNull = true; - pCtx[i].nStartQueryTimestamp = timestamp; - pCtx[i].aInputElemBuf = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page); - - // in case of tag column, the tag information should be extracted from input buffer - if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TAG) { - tVariantDestroy(&pCtx[i].tag); - - int32_t type = pCtx[i].outputType; - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - tVariantCreateFromBinary(&pCtx[i].tag, varDataVal(pCtx[i].aInputElemBuf), varDataLen(pCtx[i].aInputElemBuf), type); - } else { - tVariantCreateFromBinary(&pCtx[i].tag, pCtx[i].aInputElemBuf, pCtx[i].inputBytes, pCtx[i].inputType); - } - - } - } - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionId = pQuery->pExpr1[i].base.functionId; - if (functionId == TSDB_FUNC_TAG_DUMMY) { - continue; - } - - aAggs[functionId].distMergeFunc(&pCtx[i]); - } -} - static UNUSED_FUNC void printBinaryData(int32_t functionId, char *data, int32_t srcDataType) { if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_LAST_DST) { switch (srcDataType) { @@ -3170,8 +3115,8 @@ int32_t mergeGroupResult(SQInfo *pQInfo) { SArray *group = GET_TABLEGROUP(pQInfo, pQInfo->groupIndex); int32_t ret = mergeIntoGroupResultImpl(pGroupResInfo, group, pQInfo); - if (ret < 0) { - return -1; + if (ret != TSDB_CODE_SUCCESS) { + return ret; } // this group generates at least one result, return results @@ -3266,7 +3211,7 @@ int32_t mergeIntoGroupResultImpl(SGroupResInfo* pGroupResInfo, SArray *pTableLis posList = calloc(size, sizeof(int32_t)); pTableQueryInfoList = malloc(POINTER_BYTES * size); - if (pTableQueryInfoList == NULL || posList == NULL) { + if (pTableQueryInfoList == NULL || posList == NULL || pGroupResInfo->pRows == NULL) { qError("QInfo:%p failed alloc memory", pQInfo); code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _end; @@ -3356,10 +3301,6 @@ int32_t mergeIntoGroupResultImpl(SGroupResInfo* pGroupResInfo, SArray *pTableLis tfree(posList); tfree(pTree); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, code); - } - return code; } @@ -3368,14 +3309,6 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * return; } - // order has changed already - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - if (pTableQueryInfo->lastKey == pTableQueryInfo->win.skey) { - // do nothing, no results - } else {// NOTE: even win.skey != lastKey, the results may not generated. - pTableQueryInfo->win.ekey = pTableQueryInfo->lastKey + step; - } - SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; @@ -3419,7 +3352,7 @@ void disableFuncInReverseScan(SQInfo *pQInfo) { // group by normal columns and interval query on normal table SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; - if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { + if (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery)) { disableFuncInReverseScanImpl(pRuntimeEnv, pWindowResInfo, order); } else { // for simple result of table query, for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { // todo refactor @@ -3607,7 +3540,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; bool toContinue = false; - if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { + if (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery)) { // for each group result, call the finalize function for each column SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; @@ -3658,10 +3591,6 @@ static SQueryStatusInfo getQueryStatusInfo(SQueryRuntimeEnv *pRuntimeEnv, TSKEY }; TIME_WINDOW_COPY(info.w, pQuery->window); - TIME_WINDOW_COPY(info.curWindow, pTableQueryInfo->win); - - info.curWindow.skey = start; - return info; } @@ -3677,9 +3606,7 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI } // reverse order time range - pQuery->window = pStatus->curWindow; SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); - SWITCH_ORDER(pQuery->order.order); if (QUERY_IS_ASC_QUERY(pQuery)) { @@ -3745,9 +3672,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { // store the start query position SQueryStatusInfo qstatus = getQueryStatusInfo(pRuntimeEnv, start); - SET_MASTER_SCAN_FLAG(pRuntimeEnv); - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); while (1) { doScanAllDataBlocks(pRuntimeEnv); @@ -3756,13 +3681,9 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { qstatus.status = pQuery->status; // do nothing if no data blocks are found qualified during scan - if (qstatus.lastKey != pTableQueryInfo->lastKey) { - qstatus.curWindow.ekey = pTableQueryInfo->lastKey - step; - } else { // the lastkey does not increase, which means no data checked yet + if (qstatus.lastKey == pTableQueryInfo->lastKey) { qDebug("QInfo:%p no results generated in this scan", pQInfo); } - - qstatus.lastKey = pTableQueryInfo->lastKey; } if (!needScanDataBlocksAgain(pRuntimeEnv)) { @@ -3778,7 +3699,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } - STsdbQueryCond cond = createTsdbQueryCond(pQuery, &qstatus.curWindow); + STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); restoreTimeWindow(&pQInfo->tableGroupInfo, &cond); pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); if (pRuntimeEnv->pSecQueryHandle == NULL) { @@ -3814,10 +3735,10 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { + if (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery)) { // for each group result, call the finalize function for each column SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; - if (pRuntimeEnv->groupbyNormalCol) { + if (pRuntimeEnv->groupbyColumn) { closeAllResultRows(pWindowResInfo); } @@ -3871,7 +3792,7 @@ static STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, void pTableQueryInfo->cur.vgroupIndex = -1; // set more initial size of interval/groupby query - if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) { + if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyColumn) { int32_t initialSize = 128; int32_t code = initResultRowInfo(&pTableQueryInfo->windowResInfo, initialSize, TSDB_DATA_TYPE_INT); if (code != TSDB_CODE_SUCCESS) { @@ -4224,14 +4145,14 @@ static void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBloc SResultRowInfo * pResultRowInfo = &pTableQueryInfo->windowResInfo; pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : pDataBlockInfo->rows - 1; - if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->groupbyNormalCol) { + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->groupbyColumn) { rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, pDataBlock); } else { blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, searchFn, pDataBlock); } if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - updateResultRowIndex(pResultRowInfo, pTableQueryInfo, QUERY_IS_ASC_QUERY(pQuery)); + updateResultRowIndex(pResultRowInfo, pTableQueryInfo, QUERY_IS_ASC_QUERY(pQuery), pRuntimeEnv->timeWindowInterpo); } } @@ -4267,7 +4188,7 @@ bool queryHasRemainResForTableQuery(SQueryRuntimeEnv* pRuntimeEnv) { } else { // there are results waiting for returned to client. if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED) && - (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) && + (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery)) && (pRuntimeEnv->windowResInfo.size > 0)) { return true; } @@ -4519,8 +4440,9 @@ static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* w return key; } else { // do nothing - pQuery->window.skey = tw.skey; + pQuery->window.skey = tw.skey; pWindowResInfo->prevSKey = tw.skey; + pTableQueryInfo->lastKey = tw.skey; return tw.skey; } @@ -4530,24 +4452,12 @@ static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* w static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { SQuery *pQuery = pRuntimeEnv->pQuery; - - // get the first unclosed time window - bool assign = false; - for(int32_t i = 0; i < pRuntimeEnv->windowResInfo.size; ++i) { - if (pRuntimeEnv->windowResInfo.pResult[i]->closed) { - continue; - } - - assign = true; - *start = pRuntimeEnv->windowResInfo.pResult[i]->win.skey; - } - - if (!assign) { - *start = pQuery->current->lastKey; + if (QUERY_IS_ASC_QUERY(pQuery)) { + assert(*start <= pQuery->current->lastKey); + } else { + assert(*start >= pQuery->current->lastKey); } - assert(*start <= pQuery->current->lastKey); - // if queried with value filter, do NOT forward query start position if (pQuery->limit.offset <= 0 || pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->pFillInfo != NULL) { return true; @@ -4561,6 +4471,7 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { assert(pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL); STimeWindow w = TSWINDOW_INITIALIZER; + bool ascQuery = QUERY_IS_ASC_QUERY(pQuery); SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; STableQueryInfo *pTableQueryInfo = pQuery->current; @@ -4585,19 +4496,25 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { while (pQuery->limit.offset > 0) { STimeWindow tw = win; - if ((win.ekey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (win.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { + if ((win.ekey <= blockInfo.window.ekey && ascQuery) || (win.ekey >= blockInfo.window.skey && !ascQuery)) { pQuery->limit.offset -= 1; pWindowResInfo->prevSKey = win.skey; + + // current time window is aligned with blockInfo.window.ekey + // restart it from next data block by set prevSKey to be TSKEY_INITIAL_VAL; + if ((win.ekey == blockInfo.window.ekey && ascQuery) || (win.ekey == blockInfo.window.skey && !ascQuery)) { + pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; + } } - // current window does not ended in current data block, try next data block - getNextTimeWindow(pQuery, &tw); if (pQuery->limit.offset == 0) { *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo); return true; } + // current window does not ended in current data block, try next data block + getNextTimeWindow(pQuery, &tw); + /* * If the next time window still starts from current data block, * load the primary timestamp column first, and then find the start position for the next queried time window. @@ -4605,13 +4522,12 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { * TODO: Optimize for this cases. All data blocks are not needed to be loaded, only if the first actually required * time window resides in current data block. */ - if ((tw.skey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (tw.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { - SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); + if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) { + + SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); - if ((win.ekey > blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (win.ekey < blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { + if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) { pQuery->limit.offset -= 1; } @@ -4754,7 +4670,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo pRuntimeEnv->cur.vgroupIndex = -1; pRuntimeEnv->stableQuery = isSTableQuery; pRuntimeEnv->prevGroupId = INT32_MIN; - pRuntimeEnv->groupbyNormalCol = isGroupbyNormalCol(pQuery->pGroupbyExpr); + pRuntimeEnv->groupbyColumn = isGroupbyNormalCol(pQuery->pGroupbyExpr); if (pTsBuf != NULL) { int16_t order = (pQuery->order.order == pRuntimeEnv->pTsBuf->tsOrder) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; @@ -4774,7 +4690,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo if (!QUERY_IS_INTERVAL_QUERY(pQuery)) { int16_t type = TSDB_DATA_TYPE_NULL; - if (pRuntimeEnv->groupbyNormalCol) { // group by columns not tags; + if (pRuntimeEnv->groupbyColumn) { // group by columns not tags; type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); } else { type = TSDB_DATA_TYPE_INT; // group id @@ -4785,7 +4701,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo return code; } } - } else if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery) || (!isSTableQuery)) { + } else if (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery) || (!isSTableQuery)) { int32_t numOfResultRows = getInitialPageNum(pQInfo); getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize); code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TENMB, pQInfo); @@ -4794,7 +4710,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo } int16_t type = TSDB_DATA_TYPE_NULL; - if (pRuntimeEnv->groupbyNormalCol) { + if (pRuntimeEnv->groupbyColumn) { type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); } else { type = TSDB_DATA_TYPE_TIMESTAMP; @@ -4900,7 +4816,7 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { pQuery->current = *pTableQueryInfo; doTableQueryInfoTimeWindowCheck(pQuery, *pTableQueryInfo); - if (!pRuntimeEnv->groupbyNormalCol) { + if (!pRuntimeEnv->groupbyColumn) { setEnvForEachBlock(pQInfo, *pTableQueryInfo, &blockInfo); } @@ -5158,7 +5074,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { break; } } - } else if (pRuntimeEnv->groupbyNormalCol) { // group-by on normal columns query + } else if (pRuntimeEnv->groupbyColumn) { // group-by on normal columns query while (pQInfo->groupIndex < numOfGroups) { SArray *group = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, pQInfo->groupIndex); @@ -5488,7 +5404,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } } -static void doSaveContext(SQInfo *pQInfo) { +static int32_t doSaveContext(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; @@ -5514,9 +5430,7 @@ static void doSaveContext(SQInfo *pQInfo) { pRuntimeEnv->prevGroupId = INT32_MIN; pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); - if (pRuntimeEnv->pSecQueryHandle == NULL) { - longjmp(pRuntimeEnv->env, terrno); - } + return (pRuntimeEnv->pSecQueryHandle == NULL)? -1:0; } static void doRestoreContext(SQInfo *pQInfo) { @@ -5555,7 +5469,7 @@ static void doCloseAllTimeWindowAfterScan(SQInfo *pQInfo) { static void multiTableQueryProcess(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; + SQuery *pQuery = pRuntimeEnv->pQuery; if (pQInfo->groupIndex > 0) { /* @@ -5589,12 +5503,14 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { doCloseAllTimeWindowAfterScan(pQInfo); if (needReverseScan(pQuery)) { - doSaveContext(pQInfo); - - el = scanMultiTableDataBlocks(pQInfo); - qDebug("QInfo:%p reversed scan completed, elapsed time: %" PRId64 "ms", pQInfo, el); - - doRestoreContext(pQInfo); + int32_t code = doSaveContext(pQInfo); + if (code == TSDB_CODE_SUCCESS) { + el = scanMultiTableDataBlocks(pQInfo); + qDebug("QInfo:%p reversed scan completed, elapsed time: %" PRId64 "ms", pQInfo, el); + doRestoreContext(pQInfo); + } else { + pQInfo->code = code; + } } else { qDebug("QInfo:%p no need to do reversed scan, query completed", pQInfo); } @@ -5608,12 +5524,15 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { } if (QUERY_IS_INTERVAL_QUERY(pQuery) || isSumAvgRateQuery(pQuery)) { - if (mergeGroupResult(pQInfo) == TSDB_CODE_SUCCESS) { + int32_t code = mergeGroupResult(pQInfo); + if (code == TSDB_CODE_SUCCESS) { copyResToQueryResultBuf(pQInfo, pQuery); #ifdef _DEBUG_VIEW displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num); #endif + } else { // set the error code + pQInfo->code = code; } } else { // not a interval query copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); @@ -5623,7 +5542,6 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { qDebug("QInfo:%p points returned:%" PRId64 ", total:%" PRId64, pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); } - static char *getArithemicInputSrc(void *param, const char *name, int32_t colId) { SArithmeticSupport *pSupport = (SArithmeticSupport *) param; SExprInfo* pExprInfo = (SExprInfo*) pSupport->exprList; @@ -5677,7 +5595,7 @@ static void doSecondaryArithmeticProcess(SQuery* pQuery) { } } else { arithSup.pArithExpr = pExpr; - tExprTreeCalcTraverse(arithSup.pArithExpr->pExpr, (int32_t)pQuery->rec.rows, data[i]->data, &arithSup, TSDB_ORDER_ASC, + arithmeticTreeTraverse(arithSup.pArithExpr->pExpr, (int32_t)pQuery->rec.rows, data[i]->data, &arithSup, TSDB_ORDER_ASC, getArithemicInputSrc); } } @@ -5700,7 +5618,7 @@ static void doSecondaryArithmeticProcess(SQuery* pQuery) { * select count(*)/top(field,k)/avg(field name) from table_name [where ts>now-1a]; * select count(*) from table_name group by status_column; */ -static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { +static void tableAggregationProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pRuntimeEnv->pQuery; @@ -5725,7 +5643,7 @@ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) limitResults(pRuntimeEnv); } -static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { +static void tableProjectionProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pRuntimeEnv->pQuery; @@ -5787,10 +5705,10 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQuery *pQuery = pRuntimeEnv->pQuery; pQuery->current = pTableInfo; - TSKEY newStartKey = TSKEY_INITIAL_VAL; + TSKEY newStartKey = QUERY_IS_ASC_QUERY(pQuery)? INT64_MIN:INT64_MAX; // skip blocks without load the actual data block from file if no filter condition present - if (!pRuntimeEnv->groupbyNormalCol) { + if (!pRuntimeEnv->groupbyColumn) { skipTimeInterval(pRuntimeEnv, &newStartKey); if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0 && pRuntimeEnv->pFillInfo == NULL) { setQueryStatus(pQuery, QUERY_COMPLETED); @@ -5806,7 +5724,8 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { // skip offset result rows pQuery->rec.rows = 0; - if (pQuery->fillType == TSDB_FILL_NONE) { + // not fill or no result generated during this query + if (pQuery->fillType == TSDB_FILL_NONE || pRuntimeEnv->windowResInfo.size == 0) { // all data scanned, the group by normal column can return int32_t numOfClosed = numOfClosedResultRows(&pRuntimeEnv->windowResInfo); if (pQuery->limit.offset > numOfClosed) { @@ -5886,13 +5805,13 @@ static void tableQueryImpl(SQInfo *pQInfo) { STableQueryInfo* item = taosArrayGetP(g, 0); // group by normal column, sliding window query, interval query are handled by interval query processor - if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) { // interval (down sampling operation) + if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyColumn) { // interval (down sampling operation) tableIntervalProcess(pQInfo, item); } else if (isFixedOutputQuery(pRuntimeEnv)) { - tableFixedOutputProcess(pQInfo, item); + tableAggregationProcess(pQInfo, item); } else { // diff/add/multiply/subtract/division assert(pQuery->checkBuffer == 1); - tableMultiOutputProcess(pQInfo, item); + tableProjectionProcess(pQInfo, item); } // record the total elapsed time @@ -5908,11 +5827,11 @@ static void stableQueryImpl(SQInfo *pQInfo) { int64_t st = taosGetTimestampUs(); if (QUERY_IS_INTERVAL_QUERY(pQuery) || - (isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && (!pRuntimeEnv->groupbyNormalCol))) { + (isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && (!pRuntimeEnv->groupbyColumn))) { multiTableQueryProcess(pQInfo); } else { assert((pQuery->checkBuffer == 1 && pQuery->interval.interval == 0) || isPointInterpoQuery(pQuery) || - isFirstLastRowQuery(pQuery) || pRuntimeEnv->groupbyNormalCol); + pRuntimeEnv->groupbyColumn); sequentialTableProcess(pQInfo); } @@ -5948,13 +5867,13 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pE j += 1; } } - assert(0); - return -1; + + return INT32_MIN; // return a less than TSDB_TBNAME_COLUMN_INDEX value } bool validateExprColumnInfo(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pExprMsg, SColumnInfo* pTagCols) { int32_t j = getColumnIndexInSource(pQueryMsg, pExprMsg, pTagCols); - return j < pQueryMsg->numOfCols || j < pQueryMsg->numOfTags; + return j != INT32_MIN; } static bool validateQueryMsg(SQueryTableMsg *pQueryMsg) { @@ -5981,12 +5900,14 @@ static bool validateQueryMsg(SQueryTableMsg *pQueryMsg) { return true; } -static bool validateQuerySourceCols(SQueryTableMsg *pQueryMsg, SSqlFuncMsg** pExprMsg) { +static bool validateQuerySourceCols(SQueryTableMsg *pQueryMsg, SSqlFuncMsg** pExprMsg, SColumnInfo* pTagCols) { int32_t numOfTotal = pQueryMsg->numOfCols + pQueryMsg->numOfTags; if (pQueryMsg->numOfCols < 0 || pQueryMsg->numOfTags < 0 || numOfTotal > TSDB_MAX_COLUMNS) { qError("qmsg:%p illegal value of numOfCols %d numOfTags:%d", pQueryMsg, pQueryMsg->numOfCols, pQueryMsg->numOfTags); return false; - } else if (numOfTotal == 0) { + } + + if (numOfTotal == 0) { for(int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { SSqlFuncMsg* pFuncMsg = pExprMsg[i]; @@ -6000,6 +5921,12 @@ static bool validateQuerySourceCols(SQueryTableMsg *pQueryMsg, SSqlFuncMsg** pEx } } + for(int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { + if (!validateExprColumnInfo(pQueryMsg, pExprMsg[i], pTagCols)) { + return TSDB_CODE_QRY_INVALID_MSG; + } + } + return true; } @@ -6031,7 +5958,7 @@ static char *createTableIdList(SQueryTableMsg *pQueryMsg, char *pMsg, SArray **p * @return */ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncMsg ***pExpr, SSqlFuncMsg ***pSecStageExpr, - char **tagCond, char** tbnameCond, SColIndex **groupbyCols, SColumnInfo** tagCols) { + char **tagCond, char** tbnameCond, SColIndex **groupbyCols, SColumnInfo** tagCols, char** sql) { int32_t code = TSDB_CODE_SUCCESS; if (taosCheckVersion(pQueryMsg->version, version, 3) != 0) { @@ -6063,7 +5990,9 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks); pQueryMsg->tsOrder = htonl(pQueryMsg->tsOrder); pQueryMsg->numOfTags = htonl(pQueryMsg->numOfTags); + pQueryMsg->tbnameCondLen = htonl(pQueryMsg->tbnameCondLen); pQueryMsg->secondStageOutput = htonl(pQueryMsg->secondStageOutput); + pQueryMsg->sqlstrLen = htonl(pQueryMsg->sqlstrLen); // query msg safety check if (!validateQueryMsg(pQueryMsg)) { @@ -6158,20 +6087,11 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, code = TSDB_CODE_QRY_INVALID_MSG; goto _cleanup; } - } else { -// if (!validateExprColumnInfo(pQueryMsg, pExprMsg)) { -// return TSDB_CODE_QRY_INVALID_MSG; -// } } pExprMsg = (SSqlFuncMsg *)pMsg; } - if (!validateQuerySourceCols(pQueryMsg, *pExpr)) { - code = TSDB_CODE_QRY_INVALID_MSG; - goto _cleanup; - } - if (pQueryMsg->secondStageOutput) { pExprMsg = (SSqlFuncMsg *)pMsg; *pSecStageExpr = calloc(pQueryMsg->secondStageOutput, POINTER_BYTES); @@ -6205,10 +6125,6 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, code = TSDB_CODE_QRY_INVALID_MSG; goto _cleanup; } - } else { -// if (!validateExprColumnInfo(pQueryMsg, pExprMsg)) { -// return TSDB_CODE_QRY_INVALID_MSG; -// } } pExprMsg = (SSqlFuncMsg *)pMsg; @@ -6287,17 +6203,22 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pMsg += pQueryMsg->tagCondLen; } - if (*pMsg != 0) { - size_t len = strlen(pMsg) + 1; - - *tbnameCond = malloc(len); + if (pQueryMsg->tbnameCondLen > 0) { + *tbnameCond = calloc(1, pQueryMsg->tbnameCondLen + 1); if (*tbnameCond == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _cleanup; } - strcpy(*tbnameCond, pMsg); - pMsg += len; + strncpy(*tbnameCond, pMsg, pQueryMsg->tbnameCondLen); + pMsg += pQueryMsg->tbnameCondLen; + } + + *sql = strndup(pMsg, pQueryMsg->sqlstrLen); + + if (!validateQuerySourceCols(pQueryMsg, *pExpr, *tagCols)) { + code = TSDB_CODE_QRY_INVALID_MSG; + goto _cleanup; } qDebug("qmsg:%p query %d tables, type:%d, qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, order:%d, " @@ -6306,6 +6227,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->order, pQueryMsg->numOfOutput, pQueryMsg->numOfCols, pQueryMsg->interval.interval, pQueryMsg->fillType, pQueryMsg->tsLen, pQueryMsg->tsNumOfBlocks, pQueryMsg->limit, pQueryMsg->offset); + qDebug("qmsg:%p, sql:%s", pQueryMsg, *sql); return TSDB_CODE_SUCCESS; _cleanup: @@ -6316,6 +6238,7 @@ _cleanup: tfree(*groupbyCols); tfree(*tagCols); tfree(*tagCond); + tfree(*sql); return code; } @@ -6388,7 +6311,15 @@ static int32_t createQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t num } } else { int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); - assert(j < pQueryMsg->numOfCols || j < pQueryMsg->numOfTags); + if (TSDB_COL_IS_TAG(pExprs[i].base.colInfo.flag)) { + if (j < TSDB_TBNAME_COLUMN_INDEX || j >= pQueryMsg->numOfTags) { + return TSDB_CODE_QRY_INVALID_MSG; + } + } else { + if (j < PRIMARYKEY_TIMESTAMP_COL_INDEX || j >= pQueryMsg->numOfCols) { + return TSDB_CODE_QRY_INVALID_MSG; + } + } if (pExprs[i].base.colInfo.colId != TSDB_TBNAME_COLUMN_INDEX && j >= 0) { SColumnInfo* pCol = (TSDB_COL_IS_TAG(pExprs[i].base.colInfo.flag))? &pTagCols[j]:&pQueryMsg->colList[j]; @@ -6412,6 +6343,7 @@ static int32_t createQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t num if (pExprs[i].base.functionId == TSDB_FUNC_TAG_DUMMY || pExprs[i].base.functionId == TSDB_FUNC_TS_DUMMY) { tagLen += pExprs[i].bytes; } + assert(isValidDataType(pExprs[i].type)); } @@ -6614,7 +6546,7 @@ static void calResultBufSize(SQuery* pQuery) { } static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, - SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, bool stableQuery) { + SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, bool stableQuery, char* sql) { int16_t numOfCols = pQueryMsg->numOfCols; int16_t numOfOutput = pQueryMsg->numOfOutput; @@ -6745,6 +6677,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQInfo->arrTableIdInfo = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); pQInfo->dataReady = QUERY_RESULT_NOT_READY; pQInfo->rspContext = NULL; + pQInfo->sql = sql; pthread_mutex_init(&pQInfo->lock, NULL); tsem_init(&pQInfo->ready, 0, 0); @@ -6988,6 +6921,8 @@ static void freeQInfo(SQInfo *pQInfo) { doDestroyTableQueryInfo(&pQInfo->tableqinfoGroupInfo); tfree(pQInfo->pBuf); + tfree(pQInfo->sql); + tsdbDestroyTableGroup(&pQInfo->tableGroupInfo); taosHashCleanup(pQInfo->arrTableIdInfo); @@ -7086,6 +7021,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi int32_t code = TSDB_CODE_SUCCESS; + char *sql = NULL; char *tagCond = NULL; char *tbnameCond = NULL; SArray *pTableIdList = NULL; @@ -7098,7 +7034,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi SColumnInfo *pTagColumnInfo = NULL; SSqlGroupbyExpr *pGroupbyExpr = NULL; - code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &pSecExprMsg, &tagCond, &tbnameCond, &pGroupColIndex, &pTagColumnInfo); + code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &pSecExprMsg, &tagCond, &tbnameCond, &pGroupColIndex, &pTagColumnInfo, &sql); if (code != TSDB_CODE_SUCCESS) { goto _over; } @@ -7182,8 +7118,9 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi goto _over; } - (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, pSecExprs, &tableGroupInfo, pTagColumnInfo, isSTableQuery); + (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, pSecExprs, &tableGroupInfo, pTagColumnInfo, isSTableQuery, sql); + sql = NULL; pExprs = NULL; pSecExprs = NULL; pGroupbyExpr = NULL; @@ -7207,6 +7144,7 @@ _over: } free(pTagColumnInfo); + free(sql); free(pExprs); free(pSecExprs); diff --git a/src/query/src/qHistogram.c b/src/query/src/qHistogram.c index 6ae6ede972455ce07c657a4d215b21727c1b1db5..0d31948e59e520452605ac4577b695c53b3fad6c 100644 --- a/src/query/src/qHistogram.c +++ b/src/query/src/qHistogram.c @@ -174,7 +174,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { } assert((*pHisto)->elems[idx].val > val); - } else { + } else if ((*pHisto)->numOfElems > 0) { assert((*pHisto)->elems[(*pHisto)->numOfEntries].val < val); } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 7e2e135c4bca4bd5988cd2e40c2cf19f4b316869..2444283435efeefdccdae2da5a047f23b7330ad5 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -2645,13 +2645,12 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC return pTableGroup; } -static bool indexedNodeFilterFp(const void* pNode, void* param) { +static bool tableFilterFp(const void* pNode, void* param) { tQueryInfo* pInfo = (tQueryInfo*) param; STable* pTable = (STable*)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); - char* val = NULL; - + char* val = NULL; if (pInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { val = (char*) TABLE_NAME(pTable); } else { @@ -2706,15 +2705,17 @@ static bool indexedNodeFilterFp(const void* pNode, void* param) { return true; } +static void getTableListfromSkipList(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param); + static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) { // query according to the expression tree SExprTraverseSupp supp = { - .nodeFilterFn = (__result_filter_fn_t) indexedNodeFilterFp, + .nodeFilterFn = (__result_filter_fn_t) tableFilterFp, .setupInfoFn = filterPrepare, .pExtInfo = pSTable->tagSchema, }; - tExprTreeTraverse(pExpr, pSTable->pIndex, pRes, &supp); + getTableListfromSkipList(pExpr, pSTable->pIndex, pRes, &supp); tExprTreeDestroy(&pExpr, destroyHelper); return TSDB_CODE_SUCCESS; } @@ -2956,3 +2957,235 @@ void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) { taosArrayDestroy(pGroupList->pGroupList); pGroupList->numOfTables = 0; } + +static void applyFilterToSkipListNode(SSkipList *pSkipList, tExprNode *pExpr, SArray *pResult, SExprTraverseSupp *param) { + SSkipListIterator* iter = tSkipListCreateIter(pSkipList); + + // Scan each node in the skiplist by using iterator + while (tSkipListIterNext(iter)) { + SSkipListNode *pNode = tSkipListIterGet(iter); + if (exprTreeApplayFilter(pExpr, pNode, param)) { + taosArrayPush(pResult, &(SL_GET_NODE_DATA(pNode))); + } + } + + tSkipListDestroyIter(iter); +} + +typedef struct { + char* v; + int32_t optr; +} SEndPoint; + +typedef struct { + SEndPoint* start; + SEndPoint* end; +} SQueryCond; + +// todo check for malloc failure +static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) { + int32_t optr = queryColInfo->optr; + + if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL || + optr == TSDB_RELATION_EQUAL || optr == TSDB_RELATION_NOT_EQUAL) { + pCond->start = calloc(1, sizeof(SEndPoint)); + pCond->start->optr = queryColInfo->optr; + pCond->start->v = queryColInfo->q; + } else if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { + pCond->end = calloc(1, sizeof(SEndPoint)); + pCond->end->optr = queryColInfo->optr; + pCond->end->v = queryColInfo->q; + } else if (optr == TSDB_RELATION_IN || optr == TSDB_RELATION_LIKE) { + assert(0); + } + + return TSDB_CODE_SUCCESS; +} + +static void queryIndexedColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* result) { + SSkipListIterator* iter = NULL; + + SQueryCond cond = {0}; + if (setQueryCond(pQueryInfo, &cond) != TSDB_CODE_SUCCESS) { + //todo handle error + } + + if (cond.start != NULL) { + iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_ASC); + } else { + iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->type, TSDB_ORDER_DESC); + } + + if (cond.start != NULL) { + int32_t optr = cond.start->optr; + + if (optr == TSDB_RELATION_EQUAL) { // equals + while(tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + + int32_t ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v); + if (ret != 0) { + break; + } + + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); + } + } else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) { // greater equal + bool comp = true; + int32_t ret = 0; + + while(tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + + if (comp) { + ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v); + assert(ret >= 0); + } + + if (ret == 0 && optr == TSDB_RELATION_GREATER) { + continue; + } else { + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); + comp = false; + } + } + } else if (optr == TSDB_RELATION_NOT_EQUAL) { // not equal + bool comp = true; + + while(tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0); + if (comp) { + continue; + } + + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); + } + + tSkipListDestroyIter(iter); + + comp = true; + iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_DESC); + while(tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0); + if (comp) { + continue; + } + + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); + } + + } else { + assert(0); + } + } else { + int32_t optr = cond.end ? cond.end->optr : TSDB_RELATION_INVALID; + if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { + bool comp = true; + int32_t ret = 0; + + while (tSkipListIterNext(iter)) { + SSkipListNode *pNode = tSkipListIterGet(iter); + + if (comp) { + ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.end->v); + assert(ret <= 0); + } + + if (ret == 0 && optr == TSDB_RELATION_LESS) { + continue; + } else { + STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); + comp = false; // no need to compare anymore + } + } + } else { + assert(pQueryInfo->optr == TSDB_RELATION_ISNULL || pQueryInfo->optr == TSDB_RELATION_NOTNULL); + + while (tSkipListIterNext(iter)) { + SSkipListNode *pNode = tSkipListIterGet(iter); + + bool isnull = isNull(SL_GET_NODE_KEY(pSkipList, pNode), pQueryInfo->sch.type); + if ((pQueryInfo->optr == TSDB_RELATION_ISNULL && isnull) || + (pQueryInfo->optr == TSDB_RELATION_NOTNULL && (!isnull))) { + STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); + } + } + } + } + + free(cond.start); + free(cond.end); + tSkipListDestroyIter(iter); +} + +static void queryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* res, __result_filter_fn_t filterFp) { + SSkipListIterator* iter = tSkipListCreateIter(pSkipList); + + while (tSkipListIterNext(iter)) { + bool addToResult = false; + + SSkipListNode *pNode = tSkipListIterGet(iter); + + char *pData = SL_GET_NODE_DATA(pNode); + tstr *name = (tstr*) tsdbGetTableName((void*) pData); + + // todo speed up by using hash + if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { + if (pQueryInfo->optr == TSDB_RELATION_IN) { + addToResult = pQueryInfo->compare(name, pQueryInfo->q); + } else if (pQueryInfo->optr == TSDB_RELATION_LIKE) { + addToResult = !pQueryInfo->compare(name, pQueryInfo->q); + } + } else { + addToResult = filterFp(pNode, pQueryInfo); + } + + if (addToResult) { + STableKeyInfo info = {.pTable = (void*)pData, .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(res, &info); + } + } + + tSkipListDestroyIter(iter); +} + +// Apply the filter expression to each node in the skiplist to acquire the qualified nodes in skip list +void getTableListfromSkipList(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param) { + if (pExpr == NULL) { + return; + } + + tExprNode *pLeft = pExpr->_node.pLeft; + tExprNode *pRight = pExpr->_node.pRight; + + // column project + if (pLeft->nodeType != TSQL_NODE_EXPR && pRight->nodeType != TSQL_NODE_EXPR) { + assert(pLeft->nodeType == TSQL_NODE_COL && (pRight->nodeType == TSQL_NODE_VALUE || pRight->nodeType == TSQL_NODE_DUMMY)); + + param->setupInfoFn(pExpr, param->pExtInfo); + + tQueryInfo *pQueryInfo = pExpr->_node.info; + if (pQueryInfo->indexed && pQueryInfo->optr != TSDB_RELATION_LIKE) { + queryIndexedColumn(pSkipList, pQueryInfo, result); + } else { + queryIndexlessColumn(pSkipList, pQueryInfo, result, param->nodeFilterFn); + } + + return; + } + + // The value of hasPK is always 0. + uint8_t weight = pLeft->_node.hasPK + pRight->_node.hasPK; + assert(weight == 0 && pSkipList != NULL && taosArrayGetSize(result) == 0); + + //apply the hierarchical filter expression to every node in skiplist to find the qualified nodes + applyFilterToSkipListNode(pSkipList, pExpr, result, param); +} diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index 7ec4e5445aa2fed72bb7962bcec4c7da159a7afb..b6b49693f60b15a51eaa35c95555d0f35d36f017 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -130,16 +130,14 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param); +void taosHashEmpty(SHashObj *pHashObj); + /** * clean up hash table * @param handle */ void taosHashCleanup(SHashObj *pHashObj); -/* -void *SHashMutableIterator* taosHashCreateIter(SHashObj *pHashObj, void *); -*/ - /** * * @param pHashObj diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 2af25a7b3a68b694c9aa9e5db33e7b9a13db48b8..1a9c6d314b3fd8a6b3ad6928b6faa4f7fad32757 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -313,10 +313,10 @@ void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void } if (d != NULL) { - memcpy(d, GET_HASH_NODE_DATA(pNode), dsize); - } else { - data = GET_HASH_NODE_DATA(pNode); + memcpy(d, GET_HASH_NODE_DATA(pNode), pNode->dataLen); } + + data = GET_HASH_NODE_DATA(pNode); } if (pHashObj->type == HASH_ENTRY_LOCK) { @@ -472,38 +472,49 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi return 0; } -void taosHashCleanup(SHashObj *pHashObj) { +void taosHashEmpty(SHashObj *pHashObj) { if (pHashObj == NULL) { return; } + uDebug("hash:%p cleanup hash table", pHashObj); + SHashNode *pNode, *pNext; __wr_lock(&pHashObj->lock, pHashObj->type); - if (pHashObj->hashList) { - for (int32_t i = 0; i < pHashObj->capacity; ++i) { - SHashEntry *pEntry = pHashObj->hashList[i]; - if (pEntry->num == 0) { - assert(pEntry->next == 0); - continue; - } + for (int32_t i = 0; i < pHashObj->capacity; ++i) { + SHashEntry *pEntry = pHashObj->hashList[i]; + if (pEntry->num == 0) { + assert(pEntry->next == 0); + continue; + } - pNode = pEntry->next; - assert(pNode != NULL); + pNode = pEntry->next; + assert(pNode != NULL); - while (pNode) { - pNext = pNode->next; - FREE_HASH_NODE(pHashObj, pNode); + while (pNode) { + pNext = pNode->next; + FREE_HASH_NODE(pHashObj, pNode); - pNode = pNext; - } + pNode = pNext; } - free(pHashObj->hashList); + pEntry->num = 0; + pEntry->next = NULL; } + pHashObj->size = 0; __wr_unlock(&pHashObj->lock, pHashObj->type); +} + +void taosHashCleanup(SHashObj *pHashObj) { + if (pHashObj == NULL) { + return; + } + + taosHashEmpty(pHashObj); + tfree(pHashObj->hashList); // destroy mem block size_t memBlock = taosArrayGetSize(pHashObj->pMemBlock); diff --git a/tests/script/general/parser/first_last.sim b/tests/script/general/parser/first_last.sim index 773f92afcfd1fb126f137befd9de1107b050356f..df9a4598e0ba5438e45d299d467d86b2989eb825 100644 --- a/tests/script/general/parser/first_last.sim +++ b/tests/script/general/parser/first_last.sim @@ -106,7 +106,7 @@ while $x < 5000 endw system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 1000 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect diff --git a/tests/script/general/parser/function.sim b/tests/script/general/parser/function.sim index 3505ad1a28139a85e62193eddeb050dfde0bbc0d..78b63c2baf460907a2d88836c52f157a8753c89c 100644 --- a/tests/script/general/parser/function.sim +++ b/tests/script/general/parser/function.sim @@ -244,3 +244,120 @@ if $data00 != -2.000000000 then print expect -2.000000000, actual: $data00 return -1 endi + +sql create table tm1 (ts timestamp, k int); +sql insert into tm1 values('2020-10-30 18:11:56.680', -1000); +sql insert into tm1 values('2020-11-19 18:11:45.773', NULL); +sql insert into tm1 values('2020-12-09 18:11:17.098', NULL); +sql insert into tm1 values('2020-12-20 18:11:49.412', 1); +sql insert into tm1 values('2020-12-23 18:11:50.412', 2); +sql insert into tm1 values('2020-12-28 18:11:52.412', 3); + +print =====================> td-2610 +sql select twa(k)from tm1 where ts>='2020-11-19 18:11:45.773' and ts<='2020-12-9 18:11:17.098' +if $rows != 0 then + return -1 +endi + +print =====================> td-2609 +sql select apercentile(k, 50) from tm1 where ts>='2020-10-30 18:11:56.680' and ts<='2020-12-09 18:11:17.098' +if $rows != 1 then + return -1 +endi + +if $data00 != -1000.000000000 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 1000 +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sql connect +sleep 500 + +sql use m_func_db0 + +print =====================> td-2583 +sql select min(k) from tm1 where ts>='2020-11-19 18:11:45.773' and ts<='2020-12-20 18:11:49.412' +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + print expect 1, actual: $data00 + return -1 +endi + +print =====================> td-2601 +sql select count(*) from tm1 where ts<='2020-6-1 00:00:00' and ts>='2020-1-1 00:00:00' interval(1n) fill(NULL) +if $rows != 0 then + return -1 +endi + +print =====================> td-2615 +sql select last(ts) from tm1 interval(17a) limit 776 offset 3 +if $rows != 3 then + return -1 +endi + +sql select last(ts) from tm1 interval(17a) limit 1000 offset 4 +if $rows != 2 then + return -1 +endi + +sql select last(ts) from tm1 interval(17a) order by ts desc limit 1000 offset 0 +if $rows != 6 then + return -1 +endi + +print ==================> td-2624 +sql create table tm2(ts timestamp, k int, b binary(12)); +sql insert into tm2 values('2011-01-02 18:42:45.326', -1,'abc'); +sql insert into tm2 values('2020-07-30 17:44:06.283', 0, null); +sql insert into tm2 values('2020-07-30 17:44:19.578', 9999999, null); +sql insert into tm2 values('2020-07-30 17:46:06.417', NULL, null); +sql insert into tm2 values('2020-11-09 18:42:25.538', 0, null); +sql insert into tm2 values('2020-12-29 17:43:11.641', 0, null); +sql insert into tm2 values('2020-12-29 18:43:17.129', 0, null); +sql insert into tm2 values('2020-12-29 18:46:19.109', NULL, null); +sql insert into tm2 values('2021-01-03 18:40:40.065', 0, null); + +sql select twa(k),first(ts) from tm2 where k <50 interval(17s); +if $rows != 6 then + return -1 +endi + +if $data00 != @11-01-02 18:42:42.000@ then + return -1 +endi + +if $data02 != @11-01-02 18:42:45.326@ then + return -1 +endi + +if $data10 != @20-07-30 17:43:59.000@ then + return -1 +endi + +if $data21 != 0.000000000 then + return -1 +endi + +sql select twa(k),first(ts) from tm2 where k <50 interval(17s) order by ts desc; +if $rows != 6 then + return -1 +endi + +sql select twa(k),first(ts),count(k),first(k) from tm2 interval(17s) limit 20 offset 0; +if $rows != 9 then + return -1 +endi + +if $data00 != @11-01-02 18:42:42.000@ then + return -1 +endi + +if $data10 != @20-07-30 17:43:59.000@ then + return -1 +endi