diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index ecbef79ae4ef5d4efe00971634b29da0da73e366..ac1894369d991a7332191803c9f374e63628d6d3 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -48,7 +48,7 @@ ELSEIF (TD_WINDOWS_64) IF (NOT TD_GODLL) SET_TARGET_PROPERTIES(taos PROPERTIES LINK_FLAGS /DEF:${TD_COMMUNITY_DIR}/src/client/src/taos.def) ENDIF () - TARGET_LINK_LIBRARIES(taos trpc) + TARGET_LINK_LIBRARIES(taos trpc tutil) ELSEIF (TD_DARWIN_64) SET(CMAKE_MACOSX_RPATH 1) diff --git a/src/client/inc/tscSecondaryMerge.h b/src/client/inc/tscSecondaryMerge.h index 5370d0ec5259c3e47e065d4fe8b37884bdf6a050..f6c75167547442c59a2b74623a6f480734768327 100644 --- a/src/client/inc/tscSecondaryMerge.h +++ b/src/client/inc/tscSecondaryMerge.h @@ -120,7 +120,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd void tscDestroyLocalReducer(SSqlObj *pSql); -int32_t tscDoLocalreduce(SSqlObj *pSql); +int32_t tscDoLocalMerge(SSqlObj *pSql); #ifdef __cplusplus } diff --git a/src/client/inc/tscSubquery.h b/src/client/inc/tscSubquery.h index f8a6fbf5b11d688b16bf7625771b1e893bed52c1..368fe2250a37e3ef226174b80a39e63e0c990fda 100644 --- a/src/client/inc/tscSubquery.h +++ b/src/client/inc/tscSubquery.h @@ -29,8 +29,8 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql); int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql); void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code); -SJoinSubquerySupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, int32_t index); -void tscDestroyJoinSupporter(SJoinSubquerySupporter* pSupporter); +SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, int32_t index); +void tscDestroyJoinSupporter(SJoinSupporter* pSupporter); int32_t tscHandleMasterJoinQuery(SSqlObj* pSql); @@ -38,6 +38,9 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql); int32_t tscHandleMultivnodeInsert(SSqlObj *pSql); +void tscBuildResFromSubqueries(SSqlObj *pSql); +void **doSetResultRowData(SSqlObj *pSql, bool finalResult); + #ifdef __cplusplus } #endif diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 718dfcf475136aab944c0b9c889a35d0f48c2bba..2b53342e74feece2ecf2920e79234ef9cd52fb35 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -51,22 +51,38 @@ typedef struct SParsedDataColInfo { bool hasVal[TSDB_MAX_COLUMNS]; } SParsedDataColInfo; -typedef struct SJoinSubquerySupporter { +typedef struct STidTags { + int64_t uid; + int32_t tid; + int32_t vgId; + char tag[]; +} STidTags; + +typedef struct SJoinSupporter { SSubqueryState* pState; SSqlObj* pObj; // parent SqlObj int32_t subqueryIndex; // index of sub query int64_t interval; // interval time SLimitVal limit; // limit info uint64_t uid; // query meter uid - SArray* colList; // previous query information - SArray* exprsInfo; + SArray* colList; // previous query information, no need to use this attribute, and the corresponding attribution + SArray* exprList; SFieldInfo fieldsInfo; STagCond tagCond; SSqlGroupbyExpr groupbyExpr; struct STSBuf* pTSBuf; // the TSBuf struct that holds the compressed timestamp array FILE* f; // temporary file in order to create TSBuf - char path[PATH_MAX]; // temporary file path -} SJoinSubquerySupporter; + char path[PATH_MAX]; // temporary file path, todo dynamic allocate memory + int32_t tagSize; // the length of each in the first filter stage + char* pIdTagList; // result of first stage tags + int32_t totalLen; + int32_t num; +} SJoinSupporter; + +typedef struct SVgroupTableInfo { + SCMVgroupInfo vgInfo; + SArray* itemList; //SArray +} SVgroupTableInfo; int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, const char* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks); @@ -87,7 +103,7 @@ int32_t tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList, int32_t startOffset, int32_t rowSize, const char* tableId, STableMeta* pTableMeta, STableDataBlocks** dataBlocks); -UNUSED_FUNC STableIdInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx); +//UNUSED_FUNC STableIdInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx); /** * @@ -120,7 +136,7 @@ void addRequiredTagColumn(STableMetaInfo* pTableMetaInfo, SColumnIndex* index); int32_t tscSetTableId(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql); void tscClearInterpInfo(SQueryInfo* pQueryInfo); -bool tscIsInsertOrImportData(char* sqlstr); +bool tscIsInsertData(char* sqlstr); /* use for keep current db info temporarily, for handle table with db prefix */ // todo remove it @@ -160,7 +176,7 @@ SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functi int32_t tscSqlExprNumOfExprs(SQueryInfo* pQueryInfo); SSqlExpr* tscSqlExprGet(SQueryInfo* pQueryInfo, int32_t index); -SArray* tscSqlExprCopy(const SArray* src, uint64_t uid, bool deepcopy); +void tscSqlExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy); void tscSqlExprInfoDestroy(SArray* pExprInfo); SColumn* tscColumnClone(const SColumn* src); @@ -190,7 +206,6 @@ bool tscShouldFreeHeatBeat(SSqlObj* pHb); void tscCleanSqlCmd(SSqlCmd* pCmd); bool tscShouldBeFreed(SSqlObj* pSql); -void tscClearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache); STableMetaInfo* tscGetTableMetaInfoFromCmd(SSqlCmd *pCmd, int32_t subClauseIndex, int32_t tableIndex); STableMetaInfo* tscGetMetaInfo(SQueryInfo *pQueryInfo, int32_t tableIndex); @@ -204,7 +219,10 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo *pQueryInfo); int32_t tscAddSubqueryInfo(SSqlCmd *pCmd); + void tscFreeQueryInfo(SSqlCmd* pCmd); +void tscInitQueryInfo(SQueryInfo* pQueryInfo); + void tscClearSubqueryInfo(SSqlCmd* pCmd); int tscGetSTableVgroupInfo(SSqlObj* pSql, int32_t clauseIndex); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index d04fa9900d1c4915fde7c181bad8cbe3d9dc0686..7cb9c513e5e504212ac2ec47e3b1d93a21a6279f 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -45,8 +45,6 @@ enum { DATA_FROM_DATA_FILE = 2, }; -typedef SCMSTableVgroupRspMsg SVgroupsInfo; - typedef struct STableComInfo { uint8_t numOfTags; uint8_t precision; @@ -69,12 +67,13 @@ typedef struct STableMeta { } STableMeta; typedef struct STableMetaInfo { - STableMeta * pTableMeta; // table meta, cached in client side and acquried by name + STableMeta * pTableMeta; // table meta, cached in client side and acquired by name SVgroupsInfo *vgroupList; - + SArray *pVgroupTables; // SArray + /* - * 1. keep the vnode index during the multi-vnode super table projection query - * 2. keep the vnode index for multi-vnode insertion + * 1. keep the vgroup index during the multi-vnode super table projection query + * 2. keep the vgroup index for multi-vnode insertion */ int32_t vgroupIndex; char name[TSDB_TABLE_ID_LEN]; // (super) table name @@ -102,7 +101,7 @@ typedef struct SColumnIndex { typedef struct SFieldSupInfo { bool visible; - SArithExprInfo *pArithExprInfo; + SExprInfo *pArithExprInfo; SSqlExpr * pSqlExpr; } SFieldSupInfo; @@ -206,7 +205,7 @@ typedef struct SQueryInfo { SArray * colList; // SArray SFieldInfo fieldsInfo; - SArray * exprsInfo; // SArray + SArray * exprList; // SArray SLimitVal limit; SLimitVal slimit; STagCond tagCond; @@ -382,7 +381,6 @@ int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo); void tscDestroyResPointerInfo(SSqlRes *pRes); void tscResetSqlCmdObj(SSqlCmd *pCmd); -void tscFreeResData(SSqlObj *pSql); /** * free query result of the sql object @@ -395,7 +393,7 @@ void tscFreeSqlResult(SSqlObj *pSql); * Note: this function is multi-thread safe. * @param pObj */ -void tscFreeSqlObjPartial(SSqlObj *pObj); +void tscPartiallyFreeSqlObj(SSqlObj *pObj); /** * free sql object, release allocated resource @@ -423,7 +421,7 @@ int32_t tscInvalidSQLErrMsg(char *msg, const char *additionalInfo, const char *s void tscQueueAsyncFreeResult(SSqlObj *pSql); int32_t tscToSQLCmd(SSqlObj *pSql, struct SSqlInfo *pInfo); -char * tscGetResultColumnChr(SSqlRes *pRes, SQueryInfo *pQueryInfo, int32_t column); +char * tscGetResultColumnChr(SSqlRes *pRes, SQueryInfo *pQueryInfo, int32_t column, int16_t bytes); extern void * pVnodeConn; extern void * tscCacheHandle; diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 3dbf74eaa7fbf37a8c94faf4787a8082cc4fc0ed..b64c1ed8c0446c26276f90846ed5883063f047f0 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -14,17 +14,18 @@ */ #include "os.h" +#include "tutil.h" + +#include "tnote.h" #include "trpc.h" #include "tscLog.h" #include "tscProfile.h" +#include "tscSubquery.h" #include "tscSecondaryMerge.h" #include "tscUtil.h" -#include "tsclient.h" -#include "tsocket.h" -#include "tutil.h" -#include "tnote.h" #include "tsched.h" #include "tschemautil.h" +#include "tsclient.h" static void tscProcessFetchRow(SSchedMsg *pMsg); static void tscAsyncQueryRowsForNextVnode(void *param, TAOS_RES *tres, int numOfRows); @@ -44,9 +45,9 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const SSqlRes *pRes = &pSql->res; pSql->signature = pSql; - pSql->param = param; - pSql->pTscObj = pObj; - pSql->maxRetry = TSDB_MAX_REPLICA_NUM; + pSql->param = param; + pSql->pTscObj = pObj; + pSql->maxRetry = TSDB_MAX_REPLICA_NUM; pSql->fp = fp; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { @@ -145,7 +146,7 @@ static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows) { } // local reducer has handle this situation during super table non-projection query. - if (pCmd->command != TSDB_SQL_RETRIEVE_METRIC) { + if (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE) { pRes->numOfTotalInCurrentClause += pRes->numOfRows; } @@ -175,7 +176,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo } pSql->fp = fp; - if (pCmd->command != TSDB_SQL_RETRIEVE_METRIC && pCmd->command < TSDB_SQL_LOCAL) { + if (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE && pCmd->command < TSDB_SQL_LOCAL) { pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; } tscProcessSql(pSql); @@ -219,12 +220,17 @@ void taos_fetch_rows_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, int), voi pSql->param = param; tscResetForNextRetrieve(pRes); - - if (pCmd->command != TSDB_SQL_RETRIEVE_METRIC && pCmd->command < TSDB_SQL_LOCAL) { - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; + + // handle the sub queries of join query + if (pCmd->command == TSDB_SQL_METRIC_JOIN_RETRIEVE) { + tscFetchDatablockFromSubquery(pSql); + } else { + if (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE && pCmd->command < TSDB_SQL_LOCAL) { + pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; + } + + tscProcessSql(pSql); } - - tscProcessSql(pSql); } void taos_fetch_row_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, TAOS_ROW), void *param) { @@ -251,7 +257,7 @@ void taos_fetch_row_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, TAOS_ROW), tscResetForNextRetrieve(pRes); pSql->fp = tscAsyncFetchSingleRowProxy; - if (pCmd->command != TSDB_SQL_RETRIEVE_METRIC && pCmd->command < TSDB_SQL_LOCAL) { + if (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE && pCmd->command < TSDB_SQL_LOCAL) { pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; } @@ -311,7 +317,7 @@ void tscProcessFetchRow(SSchedMsg *pMsg) { SFieldSupInfo* pSup = taosArrayGet(pQueryInfo->fieldsInfo.pSupportInfo, i); if (pSup->pSqlExpr != NULL) { - pRes->tsrow[i] = tscGetResultColumnChr(pRes, pQueryInfo, i); + pRes->tsrow[i] = tscGetResultColumnChr(pRes, pQueryInfo, i, pSup->pSqlExpr->resBytes); } else { // todo add } diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index bc6231ba09645267e8206293d47d3eff27c9c668..caaedca1f297a91fde376d41f2f63be118c43618 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -153,7 +153,7 @@ typedef struct SRateInfo { int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type, - int16_t *bytes, int16_t *intermediateResBytes, int16_t extLength, bool isSuperTable) { + int16_t *bytes, int16_t *interResBytes, int16_t extLength, bool isSuperTable) { if (!isValidDataType(dataType, dataBytes)) { tscError("Illegal data type %d or data type length %d", dataType, dataBytes); return TSDB_CODE_INVALID_SQL; @@ -164,28 +164,35 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_INTERP) { *type = (int16_t)dataType; *bytes = (int16_t)dataBytes; - *intermediateResBytes = *bytes + sizeof(SResultInfo); + *interResBytes = *bytes + sizeof(SResultInfo); + return TSDB_CODE_SUCCESS; + } + + if (functionId == TSDB_FUNC_TID_TAG) { // todo use struct + *type = TSDB_DATA_TYPE_BINARY; + *bytes = dataBytes + sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t); // (uid, tid) + VGID + TAGSIZE + *interResBytes = *bytes; return TSDB_CODE_SUCCESS; } if (functionId == TSDB_FUNC_COUNT) { *type = TSDB_DATA_TYPE_BIGINT; *bytes = sizeof(int64_t); - *intermediateResBytes = *bytes; + *interResBytes = *bytes; return TSDB_CODE_SUCCESS; } if (functionId == TSDB_FUNC_ARITHM) { *type = TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(double); - *intermediateResBytes = *bytes; + *interResBytes = *bytes; return TSDB_CODE_SUCCESS; } if (functionId == TSDB_FUNC_TS_COMP) { *type = TSDB_DATA_TYPE_BINARY; *bytes = sizeof(int32_t); // this results is compressed ts data - *intermediateResBytes = POINTER_BYTES; + *interResBytes = POINTER_BYTES; return TSDB_CODE_SUCCESS; } @@ -193,54 +200,54 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI if (functionId == TSDB_FUNC_MIN || functionId == TSDB_FUNC_MAX) { *type = TSDB_DATA_TYPE_BINARY; *bytes = dataBytes + DATA_SET_FLAG_SIZE; - *intermediateResBytes = *bytes; + *interResBytes = *bytes; return TSDB_CODE_SUCCESS; } else if (functionId == TSDB_FUNC_SUM) { *type = TSDB_DATA_TYPE_BINARY; *bytes = sizeof(SSumInfo); - *intermediateResBytes = *bytes; + *interResBytes = *bytes; return TSDB_CODE_SUCCESS; } else if (functionId == TSDB_FUNC_AVG) { *type = TSDB_DATA_TYPE_BINARY; *bytes = sizeof(SAvgInfo); - *intermediateResBytes = *bytes; + *interResBytes = *bytes; return TSDB_CODE_SUCCESS; } else if (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE) { *type = TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(SRateInfo); - *intermediateResBytes = sizeof(SRateInfo); + *interResBytes = sizeof(SRateInfo); return TSDB_CODE_SUCCESS; } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { *type = TSDB_DATA_TYPE_BINARY; *bytes = sizeof(STopBotInfo) + (sizeof(tValuePair) + POINTER_BYTES + extLength) * param; - *intermediateResBytes = *bytes; + *interResBytes = *bytes; return TSDB_CODE_SUCCESS; } else if (functionId == TSDB_FUNC_SPREAD) { *type = TSDB_DATA_TYPE_BINARY; *bytes = sizeof(SSpreadInfo); - *intermediateResBytes = *bytes; + *interResBytes = *bytes; return TSDB_CODE_SUCCESS; } else if (functionId == TSDB_FUNC_APERCT) { *type = TSDB_DATA_TYPE_BINARY; *bytes = sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1) + sizeof(SHistogramInfo) + sizeof(SAPercentileInfo); - *intermediateResBytes = *bytes; + *interResBytes = *bytes; return TSDB_CODE_SUCCESS; } else if (functionId == TSDB_FUNC_LAST_ROW) { *type = TSDB_DATA_TYPE_BINARY; *bytes = sizeof(SLastrowInfo) + dataBytes; - *intermediateResBytes = *bytes; + *interResBytes = *bytes; return TSDB_CODE_SUCCESS; } else if (functionId == TSDB_FUNC_TWA) { *type = TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(STwaInfo); - *intermediateResBytes = *bytes; + *interResBytes = *bytes; return TSDB_CODE_SUCCESS; } } @@ -253,57 +260,57 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI } *bytes = sizeof(int64_t); - *intermediateResBytes = sizeof(SSumInfo); + *interResBytes = sizeof(SSumInfo); return TSDB_CODE_SUCCESS; } else if (functionId == TSDB_FUNC_APERCT) { *type = TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(double); - *intermediateResBytes = + *interResBytes = sizeof(SAPercentileInfo) + sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1); return TSDB_CODE_SUCCESS; } else if (functionId == TSDB_FUNC_TWA) { *type = TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(double); - *intermediateResBytes = sizeof(STwaInfo); + *interResBytes = sizeof(STwaInfo); return TSDB_CODE_SUCCESS; } if (functionId == TSDB_FUNC_AVG) { *type = TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(double); - *intermediateResBytes = sizeof(SAvgInfo); + *interResBytes = sizeof(SAvgInfo); } else if (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE) { *type = TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(double); - *intermediateResBytes = sizeof(SRateInfo); + *interResBytes = sizeof(SRateInfo); } else if (functionId == TSDB_FUNC_STDDEV) { *type = TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(double); - *intermediateResBytes = sizeof(SStddevInfo); + *interResBytes = sizeof(SStddevInfo); } else if (functionId == TSDB_FUNC_MIN || functionId == TSDB_FUNC_MAX) { *type = (int16_t)dataType; *bytes = (int16_t)dataBytes; - *intermediateResBytes = dataBytes + DATA_SET_FLAG_SIZE; + *interResBytes = dataBytes + DATA_SET_FLAG_SIZE; } else if (functionId == TSDB_FUNC_FIRST || functionId == TSDB_FUNC_LAST) { *type = (int16_t)dataType; *bytes = (int16_t)dataBytes; - *intermediateResBytes = dataBytes + sizeof(SResultInfo); + *interResBytes = dataBytes + sizeof(SResultInfo); } else if (functionId == TSDB_FUNC_SPREAD) { *type = (int16_t)TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(double); - *intermediateResBytes = sizeof(SSpreadInfo); + *interResBytes = sizeof(SSpreadInfo); } else if (functionId == TSDB_FUNC_PERCT) { *type = (int16_t)TSDB_DATA_TYPE_DOUBLE; *bytes = (int16_t)sizeof(double); - *intermediateResBytes = (int16_t)sizeof(double); + *interResBytes = (int16_t)sizeof(double); } else if (functionId == TSDB_FUNC_LEASTSQR) { *type = TSDB_DATA_TYPE_BINARY; *bytes = TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE; // string - *intermediateResBytes = *bytes + sizeof(SResultInfo); + *interResBytes = *bytes + sizeof(SResultInfo); } else if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_LAST_DST) { *type = TSDB_DATA_TYPE_BINARY; *bytes = dataBytes + sizeof(SFirstLastInfo); - *intermediateResBytes = *bytes; + *interResBytes = *bytes; } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { *type = (int16_t)dataType; *bytes = (int16_t)dataBytes; @@ -311,11 +318,11 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI size_t size = sizeof(STopBotInfo) + (sizeof(tValuePair) + POINTER_BYTES + extLength) * param; // the output column may be larger than sizeof(STopBotInfo) - *intermediateResBytes = size; + *interResBytes = size; } else if (functionId == TSDB_FUNC_LAST_ROW) { *type = (int16_t)dataType; *bytes = (int16_t)dataBytes; - *intermediateResBytes = dataBytes + sizeof(SLastrowInfo); + *interResBytes = dataBytes + sizeof(SLastrowInfo); } else { return TSDB_CODE_INVALID_SQL; } @@ -1837,6 +1844,7 @@ static void last_row_function(SQLFunctionCtx *pCtx) { assignVal(pCtx->aOutputBuf, pData, pCtx->inputBytes, pCtx->inputType); SResultInfo *pResInfo = GET_RES_INFO(pCtx); + pResInfo->hasResult = DATA_SET_FLAG; SLastrowInfo *pInfo = (SLastrowInfo *)pResInfo->interResultBuf; pInfo->ts = pCtx->param[0].i64Key; @@ -1856,14 +1864,17 @@ static void last_row_function(SQLFunctionCtx *pCtx) { static void last_row_finalizer(SQLFunctionCtx *pCtx) { // do nothing at the first stage + SResultInfo *pResInfo = GET_RES_INFO(pCtx); if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult != DATA_SET_FLAG) { setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); return; } } else { - // do nothing + if (pResInfo->hasResult != DATA_SET_FLAG) { + setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + return; + } } GET_RES_INFO(pCtx)->numOfRes = 1; @@ -2974,11 +2985,28 @@ static void tag_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { */ static void tag_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, 1, 1); - tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->tag.nType); + + char* output = pCtx->aOutputBuf; + + // todo refactor to dump length presented string(var string) + if (pCtx->tag.nType == TSDB_DATA_TYPE_BINARY || pCtx->tag.nType == TSDB_DATA_TYPE_NCHAR) { + *(int16_t*) output = pCtx->tag.nLen; + output += VARSTR_HEADER_SIZE; + } + + tVariantDump(&pCtx->tag, output, pCtx->tag.nType); } static void tag_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); + + char* output = pCtx->aOutputBuf; + + // todo refactor to dump length presented string(var string) + if (pCtx->tag.nType == TSDB_DATA_TYPE_BINARY || pCtx->tag.nType == TSDB_DATA_TYPE_NCHAR) { + *(int16_t*) output = pCtx->tag.nLen; + output += VARSTR_HEADER_SIZE; + } tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->tag.nType); } @@ -4836,7 +4864,7 @@ SQLAggFuncElem aAggs[] = {{ "apercentile", TSDB_FUNC_APERCT, TSDB_FUNC_APERCT, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_METRIC, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_STABLE, apercentile_function_setup, apercentile_function, apercentile_function_f, @@ -4881,7 +4909,7 @@ SQLAggFuncElem aAggs[] = {{ "last_row", TSDB_FUNC_LAST_ROW, TSDB_FUNC_LAST_ROW, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS | + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, first_last_function_setup, last_row_function, @@ -4897,7 +4925,7 @@ SQLAggFuncElem aAggs[] = {{ "top", TSDB_FUNC_TOP, TSDB_FUNC_TOP, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_NEED_TS | + TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, top_bottom_function_setup, top_function, @@ -4913,7 +4941,7 @@ SQLAggFuncElem aAggs[] = {{ "bottom", TSDB_FUNC_BOTTOM, TSDB_FUNC_BOTTOM, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_NEED_TS | + TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, top_bottom_function_setup, bottom_function, @@ -5079,7 +5107,7 @@ SQLAggFuncElem aAggs[] = {{ "arithmetic", TSDB_FUNC_ARITHM, TSDB_FUNC_ARITHM, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS, + TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_NEED_TS, function_setup, arithmetic_function, arithmetic_function_f, @@ -5140,7 +5168,7 @@ SQLAggFuncElem aAggs[] = {{ "interp", TSDB_FUNC_INTERP, TSDB_FUNC_INTERP, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_NEED_TS, function_setup, interp_function, do_sum_f, // todo filter handle @@ -5239,4 +5267,19 @@ SQLAggFuncElem aAggs[] = {{ sumrate_func_merge, sumrate_func_second_merge, data_req_load_info, + }, + { + // 34 + "tid_tag", // return table id and the corresponding tags for join match + TSDB_FUNC_TID_TAG, + TSDB_FUNC_TID_TAG, + TSDB_FUNCSTATE_MO, + function_setup, + noop1, + noop2, + no_next_step, + noop1, + noop1, + noop1, + data_req_load_info, }}; diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 7ace940a1e9103640c6810b9e46c4e036f27f39b..600ef8834407360a814767a0633720327fcd697c 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -131,17 +131,23 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { for (int32_t i = 0; i < numOfRows; ++i) { TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0); - strncpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i, pSchema[i].name, - TSDB_COL_NAME_LEN); + char* dst = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i; + STR_WITH_MAXSIZE_TO_VARSTR(dst, pSchema[i].name, TSDB_COL_NAME_LEN); char *type = tDataTypeDesc[pSchema[i].type].aName; pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1); - strncpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 1) * totalNumOfRows + pField->bytes * i, type, pField->bytes); - + dst = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 1) * totalNumOfRows + pField->bytes * i; + + STR_TO_VARSTR(dst, type); + int32_t bytes = pSchema[i].bytes; - if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { - bytes = bytes / TSDB_NCHAR_SIZE; + if (pSchema[i].type == TSDB_DATA_TYPE_BINARY || pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { + bytes -= VARSTR_HEADER_SIZE; + + if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { + bytes = bytes / TSDB_NCHAR_SIZE; + } } pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 2); @@ -149,8 +155,8 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 3); if (i >= tscGetNumOfColumns(pMeta) && tscGetNumOfTags(pMeta) != 0) { - strncpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i, "tag", - strlen("tag") + 1); + char* output = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i; + STR_WITH_SIZE_TO_VARSTR(output, "TAG", 3); } } @@ -163,13 +169,15 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { for (int32_t i = numOfRows; i < totalNumOfRows; ++i) { // field name TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0); - strncpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i, pSchema[i].name, - TSDB_COL_NAME_LEN); + char* output = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i; + STR_WITH_MAXSIZE_TO_VARSTR(output, pSchema[i].name, TSDB_COL_NAME_LEN); // type name pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1); char *type = tDataTypeDesc[pSchema[i].type].aName; - strncpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 1) * totalNumOfRows + pField->bytes * i, type, pField->bytes); + + output = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 1) * totalNumOfRows + pField->bytes * i; + STR_WITH_MAXSIZE_TO_VARSTR(output, type, pField->bytes); // type length int32_t bytes = pSchema[i].bytes; @@ -183,49 +191,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { // tag value pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 3); char *target = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i; - - if (isNull(pTagValue, pSchema[i].type)) { - sprintf(target, "%s", TSDB_DATA_NULL_STR); - } else { - switch (pSchema[i].type) { - case TSDB_DATA_TYPE_BINARY: - /* binary are not null-terminated string */ - strncpy(target, pTagValue, pSchema[i].bytes); - break; - case TSDB_DATA_TYPE_NCHAR: - taosUcs4ToMbs(pTagValue, pSchema[i].bytes, target); - break; - case TSDB_DATA_TYPE_FLOAT: { - float fv = 0; - fv = GET_FLOAT_VAL(pTagValue); - sprintf(target, "%f", fv); - } break; - case TSDB_DATA_TYPE_DOUBLE: { - double dv = 0; - dv = GET_DOUBLE_VAL(pTagValue); - sprintf(target, "%lf", dv); - } break; - case TSDB_DATA_TYPE_TINYINT: - sprintf(target, "%d", *(int8_t *)pTagValue); - break; - case TSDB_DATA_TYPE_SMALLINT: - sprintf(target, "%d", *(int16_t *)pTagValue); - break; - case TSDB_DATA_TYPE_INT: - sprintf(target, "%d", *(int32_t *)pTagValue); - break; - case TSDB_DATA_TYPE_BIGINT: - sprintf(target, "%" PRId64 "", *(int64_t *)pTagValue); - break; - case TSDB_DATA_TYPE_BOOL: { - char *val = (*((int8_t *)pTagValue) == 0) ? "false" : "true"; - sprintf(target, "%s", val); - break; - } - default: - break; - } - } + STR_WITH_SIZE_TO_VARSTR(target, "TAG", 3); pTagValue += pSchema[i].bytes; } @@ -233,7 +199,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { return 0; } -static int32_t tscBuildMeterSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, int32_t typeColLength, +static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, int32_t typeColLength, int32_t noteColLength) { int32_t rowLen = 0; SColumnIndex index = {0}; @@ -243,14 +209,14 @@ static int32_t tscBuildMeterSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); pQueryInfo->order.order = TSDB_ORDER_ASC; - TAOS_FIELD f = {.type = TSDB_DATA_TYPE_BINARY, .bytes = TSDB_COL_NAME_LEN}; + TAOS_FIELD f = {.type = TSDB_DATA_TYPE_BINARY, .bytes = TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE}; strncpy(f.name, "Field", TSDB_COL_NAME_LEN); SFieldSupInfo* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, TSDB_COL_NAME_LEN, - TSDB_COL_NAME_LEN, false); + pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, + TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE, TSDB_COL_NAME_LEN, false); - rowLen += TSDB_COL_NAME_LEN; + rowLen += (TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE); f.bytes = typeColLength; f.type = TSDB_DATA_TYPE_BINARY; @@ -289,137 +255,24 @@ static int32_t tscProcessDescribeTable(SSqlObj *pSql) { assert(tscGetMetaInfo(pQueryInfo, 0)->pTableMeta != NULL); - const int32_t NUM_OF_DESCRIBE_TABLE_COLUMNS = 4; + const int32_t NUM_OF_DESC_TABLE_COLUMNS = 4; const int32_t TYPE_COLUMN_LENGTH = 16; const int32_t NOTE_COLUMN_MIN_LENGTH = 8; - int32_t note_field_length = tscMaxLengthOfTagsFields(pSql); - if (note_field_length == 0) { - note_field_length = NOTE_COLUMN_MIN_LENGTH; + int32_t noteFieldLen = tscMaxLengthOfTagsFields(pSql); + if (noteFieldLen == 0) { + noteFieldLen = NOTE_COLUMN_MIN_LENGTH; } - int32_t rowLen = - tscBuildMeterSchemaResultFields(pSql, NUM_OF_DESCRIBE_TABLE_COLUMNS, TYPE_COLUMN_LENGTH, note_field_length); + int32_t rowLen = tscBuildTableSchemaResultFields(pSql, NUM_OF_DESC_TABLE_COLUMNS, TYPE_COLUMN_LENGTH, noteFieldLen); tscFieldInfoUpdateOffset(pQueryInfo); return tscSetValueToResObj(pSql, rowLen); } -// todo add order support -static int tscBuildMetricTagProjectionResult(SSqlObj *pSql) { -#if 0 - // the result structure has been completed in sql parse, so we - // only need to reorganize the results in the column format - SSqlCmd * pCmd = &pSql->cmd; - SSqlRes * pRes = &pSql->res; - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta; - SSchema * pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - - int32_t vOffset[TSDB_MAX_COLUMNS] = {0}; - - for (int32_t f = 1; f < pTableMetaInfo->numOfTags; ++f) { - int16_t tagColumnIndex = pTableMetaInfo->tagColumnIndex[f - 1]; - if (tagColumnIndex == -1) { - vOffset[f] = vOffset[f - 1] + TSDB_TABLE_NAME_LEN; - } else { - vOffset[f] = vOffset[f - 1] + pSchema[tagColumnIndex].bytes; - } - } - - int32_t totalNumOfResults = pMetricMeta->numOfTables; - int32_t rowLen = tscGetResRowLength(pQueryInfo->exprsInfo); - - tscInitResObjForLocalQuery(pSql, totalNumOfResults, rowLen); - - int32_t rowIdx = 0; - for (int32_t i = 0; i < pMetricMeta->numOfVnodes; ++i) { - SVnodeSidList *pSidList = (SVnodeSidList *)((char *)pMetricMeta + pMetricMeta->list[i]); - - for (int32_t j = 0; j < pSidList->numOfSids; ++j) { - STableIdInfo *pSidExt = tscGetMeterSidInfo(pSidList, j); - - for (int32_t k = 0; k < pQueryInfo->fieldsInfo.numOfOutput; ++k) { - SColIndex *pColIndex = &tscSqlExprGet(pQueryInfo, k)->colInfo; - int16_t offsetId = pColIndex->colIdx; - - assert((pColIndex->flag & TSDB_COL_TAG) != 0); - assert(0); - - char * val = NULL;//pSidExt->tags + vOffset[offsetId]; - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, k); - - memcpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, k) * totalNumOfResults + pField->bytes * rowIdx, val, - (size_t)pField->bytes); - } - rowIdx++; - } - } - -#endif - return 0; -} - -static int tscBuildMetricTagSqlFunctionResult(SSqlObj *pSql) { -// SSqlCmd *pCmd = &pSql->cmd; -// SSqlRes *pRes = &pSql->res; - -// SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); -#if 0 - SSuperTableMeta *pMetricMeta = tscGetMetaInfo(pQueryInfo, 0)->pMetricMeta; - int32_t totalNumOfResults = 1; // count function only produce one result - int32_t rowLen = tscGetResRowLength(pQueryInfo->exprsInfo); - - tscInitResObjForLocalQuery(pSql, totalNumOfResults, rowLen); - - int32_t rowIdx = 0; - for (int32_t i = 0; i < totalNumOfResults; ++i) { - for (int32_t k = 0; k < pQueryInfo->fieldsInfo.numOfOutput; ++k) { - SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); - - if (pExpr->colInfo.colIdx == -1 && pExpr->functionId == TSDB_FUNC_COUNT) { - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, k); - - memcpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, i) * totalNumOfResults + pField->bytes * rowIdx, - &pMetricMeta->numOfTables, sizeof(pMetricMeta->numOfTables)); - } else { - tscError("not support operations"); - continue; - } - } - rowIdx++; - } -#endif - - return 0; -} - -static int tscProcessQueryTags(SSqlObj *pSql) { - SSqlCmd *pCmd = &pSql->cmd; - - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - - STableMeta *pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; - if (pTableMeta == NULL || tscGetNumOfTags(pTableMeta) == 0 || tscGetNumOfColumns(pTableMeta) == 0) { - strcpy(pCmd->payload, "invalid table"); - pSql->res.code = TSDB_CODE_INVALID_TABLE; - return pSql->res.code; - } - - SSqlExpr *pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); - if (pExpr->functionId == TSDB_FUNC_COUNT) { - return tscBuildMetricTagSqlFunctionResult(pSql); - } else { - return tscBuildMetricTagProjectionResult(pSql); - } -} - static void tscProcessCurrentUser(SSqlObj *pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); tscSetLocalQueryResult(pSql, pSql->pTscObj->user, pExpr->aliasName, TSDB_USER_LEN); } @@ -434,7 +287,7 @@ static void tscProcessCurrentDB(SSqlObj *pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); tscSetLocalQueryResult(pSql, db, pExpr->aliasName, TSDB_DB_NAME_LEN); } @@ -442,14 +295,14 @@ static void tscProcessServerVer(SSqlObj *pSql) { const char* v = pSql->pTscObj->sversion; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); tscSetLocalQueryResult(pSql, v, pExpr->aliasName, tListLen(pSql->pTscObj->sversion)); } static void tscProcessClientVer(SSqlObj *pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); tscSetLocalQueryResult(pSql, version, pExpr->aliasName, strlen(version)); } @@ -469,7 +322,7 @@ static void tscProcessServStatus(SSqlObj *pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); tscSetLocalQueryResult(pSql, "1", pExpr->aliasName, 2); } @@ -491,7 +344,7 @@ void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnNa TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0); SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, 0); - pInfo->pSqlExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); + pInfo->pSqlExpr = taosArrayGetP(pQueryInfo->exprList, 0); strncpy(pRes->data, val, pField->bytes); } @@ -503,8 +356,6 @@ int tscProcessLocalCmd(SSqlObj *pSql) { pSql->res.code = (uint8_t)taosCfgDynamicOptions(pCmd->payload); } else if (pCmd->command == TSDB_SQL_DESCRIBE_TABLE) { pSql->res.code = (uint8_t)tscProcessDescribeTable(pSql); - } else if (pCmd->command == TSDB_SQL_RETRIEVE_TAGS) { - pSql->res.code = (uint8_t)tscProcessQueryTags(pSql); } else if (pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) { /* * set the qhandle to be 1 in order to pass the qhandle check, and to call partial release function to diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index cab7e150230a37f968ac470de34f84b6b174d3fd..8f40cf6fb9fbbfeb191e720f946fcc34890df5ee 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -305,32 +305,37 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload, case TSDB_DATA_TYPE_BINARY: // binary data cannot be null-terminated char string, otherwise the last char of the string is lost if (pToken->type == TK_NULL) { + *(int16_t*) payload = sizeof(int8_t); + payload += VARSTR_HEADER_SIZE; + *payload = TSDB_DATA_BINARY_NULL; } else { // too long values will return invalid sql, not be truncated automatically if (pToken->n > pSchema->bytes) { return tscInvalidSQLErrMsg(msg, "string data overflow", pToken->z); } - strncpy(payload, pToken->z, pToken->n); - - if (pToken->n < pSchema->bytes) { - payload[pToken->n] = 0; // add the null-terminated char if the length of the string is shorter than the available space - } + STR_WITH_SIZE_TO_VARSTR(payload, pToken->z, pToken->n); } break; case TSDB_DATA_TYPE_NCHAR: if (pToken->type == TK_NULL) { - *(uint32_t *)payload = TSDB_DATA_NCHAR_NULL; + *(int16_t*) payload = sizeof(int32_t); + payload += VARSTR_HEADER_SIZE; + + *(uint32_t*) payload = TSDB_DATA_NCHAR_NULL; } else { // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' - if (!taosMbsToUcs4(pToken->z, pToken->n, payload, pSchema->bytes)) { + int32_t resLen = -1; + if (!taosMbsToUcs4(pToken->z, pToken->n, payload + VARSTR_HEADER_SIZE, pSchema->bytes, &resLen)) { char buf[512] = {0}; snprintf(buf, 512, "%s", strerror(errno)); return tscInvalidSQLErrMsg(msg, buf, pToken->z); } + + *(uint16_t*)payload = (uint16_t) (resLen * TSDB_NCHAR_SIZE); } break; @@ -1301,13 +1306,13 @@ int tsParseSql(SSqlObj *pSql, bool initialParse) { char* p = pSql->sqlstr; pSql->sqlstr = NULL; - tscFreeSqlObjPartial(pSql); + tscPartiallyFreeSqlObj(pSql); pSql->sqlstr = p; } else { tscTrace("continue parse sql: %s", pSql->cmd.curSql); } - if (tscIsInsertOrImportData(pSql->sqlstr)) { + if (tscIsInsertData(pSql->sqlstr)) { /* * Set the fp before parse the sql string, in case of getTableMeta failed, in which * the error handle callback function can rightfully restore the user-defined callback function (fp). diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 0602fe522f42acc48646d884e7f4a972ead921b2..260649050be5448a6c975abdf1b82582a9a17575 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -300,7 +300,7 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { break; case TSDB_DATA_TYPE_NCHAR: - if (!taosMbsToUcs4(bind->buffer, *bind->length, data + param->offset, param->bytes)) { + if (!taosMbsToUcs4(bind->buffer, *bind->length, data + param->offset, param->bytes, NULL)) { return TSDB_CODE_INVALID_VALUE; } return TSDB_CODE_SUCCESS; @@ -455,7 +455,7 @@ static int insertStmtExecute(STscStmt* stmt) { // tscTrace("%p SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(taos), pObj); if (pRes->code != TSDB_CODE_SUCCESS) { - tscFreeSqlObjPartial(pSql); + tscPartiallyFreeSqlObj(pSql); } return pRes->code; @@ -510,7 +510,7 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { strtolower(sqlstr, sqlstr); pStmt->pSql->sqlstr = sqlstr; - if (tscIsInsertOrImportData(sqlstr)) { + if (tscIsInsertData(sqlstr)) { pStmt->isInsert = true; return insertStmtPrepare(pStmt); } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index dc4542a973687c12bd706a1a7755dae4acda2816..cf4aa3a50d2af09c8602882693bdae06de1232a3 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -105,7 +105,7 @@ static int32_t getColumnIndexByName(const SSQLToken* pToken, SQueryInfo* pQueryI static int32_t getTableIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); static int32_t optrToString(tSQLExpr* pExpr, char** exprString); -static int32_t getMeterIndex(SSQLToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); +static int32_t getTableIndexImpl(SSQLToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); static int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); static int32_t doLocalQueryProcess(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); static int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate); @@ -292,8 +292,8 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { break; } - case TSDB_SQL_CREATE_DNODE: { // todo parse hostname - const char* msg = "invalid ip address"; + case TSDB_SQL_CREATE_DNODE: { // todo hostname + const char* msg = "invalid host name (ip address)"; if (pInfo->pDCLInfo->nTokens > 1) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); @@ -360,14 +360,10 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } case TSDB_SQL_CFG_DNODE: { - const char* msg1 = "invalid ip address"; const char* msg2 = "invalid configure options or values"; /* validate the ip address */ tDCLSQL* pDCL = pInfo->pDCLInfo; - if (!validateIpAddress(pDCL->a[0].z, pDCL->a[0].n)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); - } /* validate the parameter names and options */ if (validateDNodeConfig(pDCL) != TSDB_CODE_SUCCESS) { @@ -1172,7 +1168,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel tExprNode* pNode = NULL; SArray* colList = taosArrayInit(10, sizeof(SColIndex)); - int32_t ret = exprTreeFromSqlExpr(&pNode, pItem->pNode, pQueryInfo->exprsInfo, pQueryInfo, colList); + int32_t ret = exprTreeFromSqlExpr(&pNode, pItem->pNode, pQueryInfo->exprList, pQueryInfo, colList); if (ret != TSDB_CODE_SUCCESS) { tExprTreeDestroy(&pNode, NULL); return invalidSqlErrMsg(pQueryInfo->msg, "invalid arithmetic expression in select clause"); @@ -1208,7 +1204,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, slot); if (pInfo->pSqlExpr == NULL) { - SArithExprInfo* pFuncExpr = calloc(1, sizeof(SArithExprInfo)); + SExprInfo* pFuncExpr = calloc(1, sizeof(SExprInfo)); pInfo->pArithExprInfo = pFuncExpr; // arithmetic expression always return result in the format of double float @@ -1219,7 +1215,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel tExprNode* pNode = NULL; // SArray* colList = taosArrayInit(10, sizeof(SColIndex)); - int32_t ret = exprTreeFromSqlExpr(&pNode, pItem->pNode, pQueryInfo->exprsInfo, pQueryInfo, NULL); + int32_t ret = exprTreeFromSqlExpr(&pNode, pItem->pNode, pQueryInfo->exprList, pQueryInfo, NULL); if (ret != TSDB_CODE_SUCCESS) { tExprTreeDestroy(&pNode, NULL); return invalidSqlErrMsg(pQueryInfo->msg, "invalid expression in select clause"); @@ -1265,14 +1261,6 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel if (isSTable) { pQueryInfo->type |= TSDB_QUERY_TYPE_STABLE_QUERY; - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - - if (tscQueryTags(pQueryInfo)) { // local handle the metric tag query - pCmd->count = numOfCols; // the number of meter schema, tricky. - pQueryInfo->command = TSDB_SQL_RETRIEVE_TAGS; - } - /* * transfer sql functions that need secondary merge into another format * in dealing with metric queries such as: count/first/last @@ -1321,12 +1309,12 @@ SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t c if (functionId == TSDB_FUNC_TAGPRJ) { index.columnIndex = colIndex - tscGetNumOfColumns(pTableMeta); - - addRequiredTagColumn(pTableMetaInfo, &index); + + tscColumnListInsert(pTableMetaInfo->tagColList, &index); pQueryInfo->type = TSDB_QUERY_TYPE_STABLE_QUERY; } else { index.columnIndex = colIndex; - pQueryInfo->type = TSDB_QUERY_TYPE_PROJECTION_QUERY; + pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; } return tscSqlExprAppend(pQueryInfo, functionId, &index, pSchema->type, pSchema->bytes, @@ -1334,7 +1322,7 @@ SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t c } void addRequiredTagColumn(STableMetaInfo* pTableMetaInfo, SColumnIndex* index) { - tscColumnListInsert(pTableMetaInfo->tagColList, index); + } static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSQLExprItem* pItem) { @@ -1375,7 +1363,7 @@ void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); if (TSDB_COL_IS_TAG(flag)) { - addRequiredTagColumn(pTableMetaInfo, pIndex); + tscColumnListInsert(pTableMetaInfo->tagColList, pIndex); } } @@ -1439,7 +1427,7 @@ int32_t addProjectionExprAndResultField(SQueryInfo* pQueryInfo, tSQLExprItem* pI } if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = TSDB_TABLE_NAME_LEN}; + SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE}; strcpy(colSchema.name, TSQL_TBNAME_L); pQueryInfo->type = TSDB_QUERY_TYPE_STABLE_QUERY; @@ -1549,7 +1537,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size, false); } else { - // count the number of meters created according to the metric + // count the number of meters created according to the super table if (getColumnIndexByName(pToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(pQueryInfo->msg, msg3); } @@ -2003,7 +1991,7 @@ int32_t doGetColumnIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColum } } -int32_t getMeterIndex(SSQLToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { +int32_t getTableIndexImpl(SSQLToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { if (pTableToken->n == 0) { // only one table and no table name prefix in column name if (pQueryInfo->numOfTables == 1) { pIndex->tableIndex = 0; @@ -2036,7 +2024,7 @@ int32_t getTableIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIn SSQLToken tableToken = {0}; extractTableNameFromToken(pToken, &tableToken); - if (getMeterIndex(&tableToken, pQueryInfo, pIndex) != TSDB_CODE_SUCCESS) { + if (getTableIndexImpl(&tableToken, pQueryInfo, pIndex) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } @@ -2343,7 +2331,7 @@ bool hasUnsupportFunctionsForSTableQuery(SQueryInfo* pQueryInfo) { size_t size = tscSqlExprNumOfExprs(pQueryInfo); for (int32_t i = 0; i < size; ++i) { int32_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; - if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_METRIC) == 0) { + if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_STABLE) == 0) { invalidSqlErrMsg(pQueryInfo->msg, msg3); return true; } @@ -2372,6 +2360,7 @@ bool hasUnsupportFunctionsForSTableQuery(SQueryInfo* pQueryInfo) { static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) { int32_t startIdx = 0; + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, startIdx); int32_t functionID = pExpr->functionId; @@ -2387,14 +2376,14 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) { size_t size = tscSqlExprNumOfExprs(pQueryInfo); for (int32_t i = startIdx + 1; i < size; ++i) { - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + SSqlExpr* pExpr1 = tscSqlExprGet(pQueryInfo, i); - int16_t functionId = pExpr->functionId; + int16_t functionId = pExpr1->functionId; if (functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TS) { continue; } - if (functionId == TSDB_FUNC_PRJ && pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + if (functionId == TSDB_FUNC_PRJ && pExpr1->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { continue; } @@ -2533,12 +2522,11 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* relIndex -= numOfCols; } - SColIndex colIndex = { - .colIndex = relIndex, .flag = TSDB_COL_TAG, .colId = pSchema->colId, - }; - + SColIndex colIndex = { .colIndex = relIndex, .flag = TSDB_COL_TAG, .colId = pSchema->colId, }; taosArrayPush(pGroupExpr->columnInfo, &colIndex); - addRequiredTagColumn(pTableMetaInfo, &index); + + index.columnIndex = relIndex; + tscColumnListInsert(pTableMetaInfo->tagColList, &index); } else { // check if the column type is valid, here only support the bool/tinyint/smallint/bigint group by if (pSchema->type > TSDB_DATA_TYPE_BINARY) { @@ -3200,7 +3188,6 @@ static bool validateJoinExprNode(SQueryInfo* pQueryInfo, tSQLExpr* pExpr, SColum const char* msg3 = "join column must have same type"; const char* msg4 = "self join is not allowed"; const char* msg5 = "join table must be the same type(table to table, super table to super table)"; - const char* msg6 = "tags in join condition not support binary/nchar types"; tSQLExpr* pRight = pExpr->pRight; @@ -3235,9 +3222,6 @@ static bool validateJoinExprNode(SQueryInfo* pQueryInfo, tSQLExpr* pExpr, SColum } else if (pLeftIndex->tableIndex == rightIndex.tableIndex) { invalidSqlErrMsg(pQueryInfo->msg, msg4); return false; - } else if (leftType == TSDB_DATA_TYPE_BINARY || leftType == TSDB_DATA_TYPE_NCHAR) { - invalidSqlErrMsg(pQueryInfo->msg, msg6); - return false; } // table to table/ super table to super table are allowed @@ -3307,7 +3291,7 @@ static int32_t handleExprInQueryCond(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, S // set join query condition if (pRight->nSQLOptr == TK_ID) { // no need to keep the timestamp join condition - pQueryInfo->type |= TSDB_QUERY_TYPE_JOIN_QUERY; + TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY); pCondExpr->tsJoin = true; /* @@ -3545,7 +3529,7 @@ static int32_t setTableCondForSTableQuery(SQueryInfo* pQueryInfo, const char* ac return TSDB_CODE_SUCCESS; } - SStringBuilder sb1 = { 0 }; + SStringBuilder sb1 = {0}; taosStringBuilderAppendStringLen(&sb1, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN); char db[TSDB_TABLE_ID_LEN] = {0}; @@ -3727,15 +3711,15 @@ static void doAddJoinTagsColumnsIntoTagList(SQueryInfo* pQueryInfo, SCondExpr* p getColumnIndexByName(&pCondExpr->pJoinExpr->pLeft->colInfo, pQueryInfo, &index); pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - -// int32_t columnInfo = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - addRequiredTagColumn(pTableMetaInfo, &index); - + + index.columnIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); + tscColumnListInsert(pTableMetaInfo->tagColList, &index); + getColumnIndexByName(&pCondExpr->pJoinExpr->pRight->colInfo, pQueryInfo, &index); pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - -// columnInfo = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - addRequiredTagColumn(pTableMetaInfo, &index); + + index.columnIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); + tscColumnListInsert(pTableMetaInfo->tagColList, &index); } } @@ -3771,7 +3755,7 @@ static int32_t getTagQueryCondExpr(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr, for(int32_t j = 0; j < num; ++j) { SColIndex* pIndex = taosArrayGet(colList, j); SColumnIndex index = {.tableIndex = i, .columnIndex = pIndex->colIndex - numOfCols}; - addRequiredTagColumn(pTableMetaInfo, &index); + tscColumnListInsert(pTableMetaInfo->tagColList, &index); } tsSetSTableQueryCond(&pQueryInfo->tagCond, uid, &bw); @@ -4052,9 +4036,7 @@ int32_t parseFillClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL) { return invalidSqlErrMsg(pQueryInfo->msg, msg); } } - - size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); - + if ((pFillToken->nExpr < size) || ((pFillToken->nExpr - 1 < size) && (tscIsPointInterpQuery(pQueryInfo)))) { tVariantListItem* lastItem = &pFillToken->a[pFillToken->nExpr - 1]; @@ -4088,7 +4070,7 @@ static void setDefaultOrderInfo(SQueryInfo* pQueryInfo) { pQueryInfo->order.orderColId = -1; } - /* for metric query, set default ascending order for group output */ + /* for super table query, set default ascending order for group output */ if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC; } @@ -4136,7 +4118,7 @@ int32_t parseOrderbyClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema SSQLToken columnName = {pVar->nLen, pVar->nType, pVar->pz}; SColumnIndex index = {0}; - if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { // metric query + if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { // super table query if (getColumnIndexByName(&columnName, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } @@ -4250,7 +4232,22 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg4 = "set tag value only available for table"; const char* msg5 = "only support add one tag"; const char* msg6 = "column can only be modified by super table"; - + + const char* msg7 = "no tags can be dropped"; + const char* msg8 = "only support one tag"; + const char* msg9 = "tag name too long"; + + const char* msg10 = "invalid tag name"; + const char* msg11 = "primary tag cannot be dropped"; + const char* msg12 = "update normal column not supported"; + const char* msg13 = "invalid tag value"; + const char* msg14 = "tag value too long"; + + const char* msg15 = "no columns can be dropped"; + const char* msg16 = "only support one column"; + const char* msg17 = "invalid column name"; + const char* msg18 = "primary timestamp column cannot be dropped"; + SSqlCmd* pCmd = &pSql->cmd; SAlterTableSQL* pAlterSQL = pInfo->pAlterInfo; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); @@ -4296,24 +4293,18 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pFieldList->p[0]); } else if (pAlterSQL->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN) { - const char* msg1 = "no tags can be dropped"; - const char* msg2 = "only support one tag"; - const char* msg3 = "tag name too long"; - const char* msg4 = "illegal tag name"; - const char* msg5 = "primary tag cannot be dropped"; - if (tscGetNumOfTags(pTableMeta) == 1) { - return invalidSqlErrMsg(pQueryInfo->msg, msg1); + return invalidSqlErrMsg(pQueryInfo->msg, msg7); } // numOfTags == 1 if (pAlterSQL->varList->nExpr > 1) { - return invalidSqlErrMsg(pQueryInfo->msg, msg2); + return invalidSqlErrMsg(pQueryInfo->msg, msg8); } tVariantListItem* pItem = &pAlterSQL->varList->a[0]; if (pItem->pVar.nLen > TSDB_COL_NAME_LEN) { - return invalidSqlErrMsg(pQueryInfo->msg, msg3); + return invalidSqlErrMsg(pQueryInfo->msg, msg9); } SColumnIndex index = COLUMN_INDEX_INITIALIZER; @@ -4324,9 +4315,9 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } if (index.columnIndex < tscGetNumOfColumns(pTableMeta)) { - return invalidSqlErrMsg(pQueryInfo->msg, msg4); + return invalidSqlErrMsg(pQueryInfo->msg, msg10); } else if (index.columnIndex == 0) { - return invalidSqlErrMsg(pQueryInfo->msg, msg5); + return invalidSqlErrMsg(pQueryInfo->msg, msg11); } char name1[128] = {0}; @@ -4335,9 +4326,6 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); } else if (pAlterSQL->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) { - const char* msg1 = "tag name too long"; - const char* msg2 = "invalid tag name"; - tVariantList* pVarList = pAlterSQL->varList; if (pVarList->nExpr > 2) { return TSDB_CODE_INVALID_SQL; @@ -4347,11 +4335,11 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { tVariantListItem* pDstItem = &pAlterSQL->varList->a[1]; if (pSrcItem->pVar.nLen >= TSDB_COL_NAME_LEN || pDstItem->pVar.nLen >= TSDB_COL_NAME_LEN) { - return invalidSqlErrMsg(pQueryInfo->msg, msg1); + return invalidSqlErrMsg(pQueryInfo->msg, msg9); } if (pSrcItem->pVar.nType != TSDB_DATA_TYPE_BINARY || pDstItem->pVar.nType != TSDB_DATA_TYPE_BINARY) { - return invalidSqlErrMsg(pQueryInfo->msg, msg2); + return invalidSqlErrMsg(pQueryInfo->msg, msg10); } SColumnIndex srcIndex = COLUMN_INDEX_INITIALIZER; @@ -4377,10 +4365,6 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); } else if (pAlterSQL->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) { - const char* msg1 = "invalid tag value"; - const char* msg2 = "update normal column not supported"; - const char* msg3 = "tag value too long"; - // Note: update can only be applied to table not super table. // the following is handle display tags value for meters created according to super table tVariantList* pVarList = pAlterSQL->varList; @@ -4393,19 +4377,19 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } if (columnIndex.columnIndex < tscGetNumOfColumns(pTableMeta)) { - return invalidSqlErrMsg(pQueryInfo->msg, msg2); + return invalidSqlErrMsg(pQueryInfo->msg, msg12); } SSchema* pTagsSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, columnIndex.columnIndex); if (tVariantDump(&pVarList->a[1].pVar, pAlterSQL->tagData.data /*pCmd->payload*/, pTagsSchema->type) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(pQueryInfo->msg, msg1); + return invalidSqlErrMsg(pQueryInfo->msg, msg13); } // validate the length of binary if ((pTagsSchema->type == TSDB_DATA_TYPE_BINARY || pTagsSchema->type == TSDB_DATA_TYPE_NCHAR) && pVarList->a[1].pVar.nLen > pTagsSchema->bytes) { - return invalidSqlErrMsg(pQueryInfo->msg, msg3); + return invalidSqlErrMsg(pQueryInfo->msg, msg14); } char name1[128] = {0}; @@ -4427,17 +4411,12 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pFieldList->p[0]); } else if (pAlterSQL->type == TSDB_ALTER_TABLE_DROP_COLUMN) { - const char* msg1 = "no columns can be dropped"; - const char* msg2 = "only support one column"; - const char* msg4 = "illegal column name"; - const char* msg3 = "primary timestamp column cannot be dropped"; - if (tscGetNumOfColumns(pTableMeta) == TSDB_MIN_COLUMNS) { // - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg15); } if (pAlterSQL->varList->nExpr > 1) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg16); } tVariantListItem* pItem = &pAlterSQL->varList->a[0]; @@ -4445,11 +4424,11 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER; SSQLToken name = {.type = TK_STRING, .z = pItem->pVar.pz, .n = pItem->pVar.nLen}; if (getColumnIndexByName(&name, pQueryInfo, &columnIndex) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(pQueryInfo->msg, msg4); + return invalidSqlErrMsg(pQueryInfo->msg, msg17); } if (columnIndex.columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidSqlErrMsg(pQueryInfo->msg, msg3); + return invalidSqlErrMsg(pQueryInfo->msg, msg18); } char name1[TSDB_COL_NAME_LEN + 1] = {0}; @@ -4469,7 +4448,7 @@ int32_t validateSqlFunctionInStreamSql(SQueryInfo* pQueryInfo) { return invalidSqlErrMsg(pQueryInfo->msg, msg0); } - size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + size_t size = taosArrayGetSize(pQueryInfo->exprList); for (int32_t i = 0; i < size; ++i) { int32_t functId = tscSqlExprGet(pQueryInfo, i)->functionId; if (!IS_STREAM_QUERY_VALID(aAggs[functId].nStatus)) { @@ -4485,7 +4464,7 @@ int32_t validateFunctionsInIntervalOrGroupbyQuery(SQueryInfo* pQueryInfo) { const char* msg1 = "column projection is not compatible with interval"; // multi-output set/ todo refactor - size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + size_t size = taosArrayGetSize(pQueryInfo->exprList); for (int32_t k = 0; k < size; ++k) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, k); @@ -4680,22 +4659,17 @@ int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL* return TSDB_CODE_SUCCESS; } + // todo refactor if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { - bool queryOnTags = false; -// if (tscQueryOnlyMetricTags(pQueryInfo, &queryOnTags) != TSDB_CODE_SUCCESS) { -// return TSDB_CODE_INVALID_SQL; -// } - - if (queryOnTags == true) { // local handle the super table tag query - pQueryInfo->command = TSDB_SQL_RETRIEVE_TAGS; - } else { + if (!tscQueryTags(pQueryInfo)) { // local handle the super table tag query if (tscIsProjectionQueryOnSTable(pQueryInfo, 0)) { if (pQueryInfo->slimit.limit > 0 || pQueryInfo->slimit.offset > 0) { return invalidSqlErrMsg(pQueryInfo->msg, msg3); } // for projection query on super table, all queries are subqueries - if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { + if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && + !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY)) { pQueryInfo->type |= TSDB_QUERY_TYPE_SUBQUERY; } } @@ -4722,6 +4696,7 @@ int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL* if (pTableMetaInfo->vgroupList->numOfVgroups == 0) { tscTrace("%p no table in super table, no output result", pSql); pQueryInfo->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; + return TSDB_CODE_SUCCESS; } // keep original limitation value in globalLimit @@ -4745,7 +4720,7 @@ int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL* return invalidSqlErrMsg(pQueryInfo->msg, msg1); } - size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + size_t size = taosArrayGetSize(pQueryInfo->exprList); // filter the query functions operating on "tbname" column that are not supported by normal columns. for (int32_t i = 0; i < size; ++i) { @@ -4857,7 +4832,7 @@ void tscAddTimestampColumn(SQueryInfo* pQueryInfo, int16_t functionId, int16_t t if (pExpr == NULL || pExpr->colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX || pExpr->functionId != functionId) { SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE, false); + pExpr = tscSqlExprInsert(pQueryInfo, 0, functionId, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE, false); pExpr->colInfo.flag = TSDB_COL_NORMAL; // NOTE: tag column does not add to source column list @@ -4872,7 +4847,7 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClau if (pParentQueryInfo->groupbyExpr.numOfGroupCols > 0) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, subClauseIndex); - size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + size_t size = taosArrayGetSize(pQueryInfo->exprList); SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, size - 1); @@ -4900,7 +4875,7 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClau pColIndex->colIndex = relIndex; index = (SColumnIndex) {.tableIndex = tableIndex, .columnIndex = relIndex}; - addRequiredTagColumn(pTableMetaInfo, &index); + tscColumnListInsert(pTableMetaInfo->tagColList, &index); } } } @@ -4941,7 +4916,7 @@ void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex) { static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) { int32_t tagLength = 0; - size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + size_t size = taosArrayGetSize(pQueryInfo->exprList); for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); @@ -4968,7 +4943,7 @@ static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) { } static void doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) { - size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + size_t size = taosArrayGetSize(pQueryInfo->exprList); for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); @@ -5006,7 +4981,7 @@ static bool onlyTagPrjFunction(SQueryInfo* pQueryInfo) { bool hasTagPrj = false; bool hasColumnPrj = false; - size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + size_t size = taosArrayGetSize(pQueryInfo->exprList); for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId == TSDB_FUNC_PRJ) { @@ -5041,7 +5016,7 @@ static bool allTagPrjInGroupby(SQueryInfo* pQueryInfo) { } static void updateTagPrjFunction(SQueryInfo* pQueryInfo) { - size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + size_t size = taosArrayGetSize(pQueryInfo->exprList); for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); @@ -5065,9 +5040,9 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo) { int16_t numOfSelectivity = 0; int16_t numOfAggregation = 0; - size_t numOfExprs = taosArrayGetSize(pQueryInfo->exprsInfo); + size_t numOfExprs = taosArrayGetSize(pQueryInfo->exprList); for (int32_t i = 0; i < numOfExprs; ++i) { - SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, i); + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, i); if (pExpr->functionId == TSDB_FUNC_TAGPRJ || (pExpr->functionId == TSDB_FUNC_PRJ && pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX)) { tagColExists = true; @@ -5076,7 +5051,7 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo) { } for (int32_t i = 0; i < numOfExprs; ++i) { - SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, i); + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, i); int16_t functionId = pExpr->functionId; if (functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TS || @@ -5125,7 +5100,7 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo) { doUpdateSqlFunctionForColPrj(pQueryInfo); } } else { - if ((pQueryInfo->type & TSDB_QUERY_TYPE_PROJECTION_QUERY) == TSDB_QUERY_TYPE_PROJECTION_QUERY) { + if ((pQueryInfo->type & TSDB_QUERY_TYPE_PROJECTION_QUERY) != 0) { if (numOfAggregation > 0 && pQueryInfo->groupbyExpr.numOfGroupCols == 0) { return invalidSqlErrMsg(pQueryInfo->msg, msg3); } @@ -5221,7 +5196,7 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { const char* msg4 = "retrieve tags not compatible with group by or interval query"; // only retrieve tags, group by is not supportted - if (pCmd->command == TSDB_SQL_RETRIEVE_TAGS) { + if (tscQueryTags(pQueryInfo)) { if (pQueryInfo->groupbyExpr.numOfGroupCols > 0 || pQueryInfo->intervalTime > 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); } else { @@ -5283,7 +5258,7 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { return TSDB_CODE_INVALID_SQL; } - // projection query on metric does not compatible with "group by" syntax + // projection query on super table does not compatible with "group by" syntax if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -5485,7 +5460,7 @@ int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* p pCmd->numOfCols = (int16_t)pFieldList->nField; - if (pTagList != NULL) { // create metric[optional] + if (pTagList != NULL) { // create super table[optional] for (int32_t i = 0; i < pTagList->nField; ++i) { tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pTagList->p[i]); } @@ -5761,6 +5736,8 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { if (code != TSDB_CODE_SUCCESS) { return code; } + } else { + TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TABLE_QUERY); } // parse the group by clause in the first place diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index c898f05a77131a4523004dc9247895bc112b0c80..35bc3c4a8081bb0cd9f13d865b5e603160702413 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -83,7 +83,6 @@ STableComInfo tscGetTableInfo(const STableMeta* pTableMeta) { return pTableMeta->tableInfo; } - bool isValidSchema(struct SSchema* pSchema, int32_t numOfCols) { if (!VALIDNUMOFCOLS(numOfCols)) { return false; @@ -184,7 +183,6 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size return pTableMeta; } - /** * the TableMeta data format in memory is as follows: * diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c index dbf17b56c5cb7893a4272d141902bf1559349ea2..1abefdfd5affe49a0ab0232736c9b4eada8a2029 100644 --- a/src/client/src/tscSecondaryMerge.c +++ b/src/client/src/tscSecondaryMerge.c @@ -288,7 +288,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd pReducer->nResultBufSize = pMemBuffer[0]->pageSize * 16; pReducer->pResultBuf = (tFilePage *)calloc(1, pReducer->nResultBufSize + sizeof(tFilePage)); - int32_t finalRowLength = tscGetResRowLength(pQueryInfo->exprsInfo); + int32_t finalRowLength = tscGetResRowLength(pQueryInfo->exprList); pReducer->resColModel = finalmodel; pReducer->resColModel->capacity = pReducer->nResultBufSize / finalRowLength; assert(finalRowLength <= pReducer->rowSize); @@ -810,7 +810,7 @@ void savePrevRecordAndSetupInterpoInfo(SLocalReducer *pLocalReducer, SQueryInfo // todo merge with following function // static void reversedCopyResultToDstBuf(SQueryInfo* pQueryInfo, SSqlRes *pRes, tFilePage *pFinalDataPage) { // -// for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { +// for (int32_t i = 0; i < pQueryInfo->exprList.numOfExprs; ++i) { // TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); // // int32_t offset = tscFieldInfoGetOffset(pQueryInfo, i); @@ -907,7 +907,7 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo savePrevRecordAndSetupInterpoInfo(pLocalReducer, pQueryInfo, &pLocalReducer->interpolationInfo); } - int32_t rowSize = tscGetResRowLength(pQueryInfo->exprsInfo); + int32_t rowSize = tscGetResRowLength(pQueryInfo->exprList); memcpy(pRes->data, pFinalDataPage->data, pRes->numOfRows * rowSize); pFinalDataPage->numOfElems = 0; @@ -1422,15 +1422,14 @@ static void doProcessResultInNextWindow(SSqlObj *pSql, int32_t numOfRes) { doExecuteSecondaryMerge(pCmd, pLocalReducer, true); } -int32_t tscDoLocalreduce(SSqlObj *pSql) { +int32_t tscDoLocalMerge(SSqlObj *pSql) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; tscResetForNextRetrieve(pRes); if (pSql->signature != pSql || pRes == NULL || pRes->pLocalReducer == NULL) { // all data has been processed - tscTrace("%s call the drop local reducer", __FUNCTION__); - + tscTrace("%p %s call the drop local reducer", pSql, __FUNCTION__); tscDestroyLocalReducer(pSql); return 0; } @@ -1441,7 +1440,7 @@ int32_t tscDoLocalreduce(SSqlObj *pSql) { // set the data merge in progress int32_t prevStatus = atomic_val_compare_exchange_32(&pLocalReducer->status, TSC_LOCALREDUCE_READY, TSC_LOCALREDUCE_IN_PROGRESS); - if (prevStatus != TSC_LOCALREDUCE_READY || pLocalReducer == NULL) { + if (prevStatus != TSC_LOCALREDUCE_READY) { assert(prevStatus == TSC_LOCALREDUCE_TOBE_FREED); // it is in tscDestroyLocalReducer function already return TSDB_CODE_SUCCESS; } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index d6f1c8f42abdef4b71fd979dc3e4d022978d7e5f..aa4eb01eae3a257ecf7d6cdb148cbea3c570eb18 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -45,15 +45,15 @@ void tscSaveSubscriptionProgress(void* sub); static int32_t minMsgSize() { return tsRpcHeadSize + 100; } -static void tscSetDnodeIpList(SSqlObj* pSql, STableMeta* pTableMeta) { +static void tscSetDnodeIpList(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) { SRpcIpSet* pIpList = &pSql->ipList; - pIpList->numOfIps = pTableMeta->vgroupInfo.numOfIps; + pIpList->numOfIps = pVgroupInfo->numOfIps; pIpList->inUse = 0; - for(int32_t i = 0; i < pTableMeta->vgroupInfo.numOfIps; ++i) { - strcpy(pIpList->fqdn[i], pTableMeta->vgroupInfo.ipAddr[i].fqdn); - pIpList->port[i] = pTableMeta->vgroupInfo.ipAddr[i].port; + for(int32_t i = 0; i < pVgroupInfo->numOfIps; ++i) { + strcpy(pIpList->fqdn[i], pVgroupInfo->ipAddr[i].fqdn); + pIpList->port[i] = pVgroupInfo->ipAddr[i].port; } } @@ -79,7 +79,7 @@ void tscSetMgmtIpListFromEdge() { if (tscMgmtIpSet.numOfIps != 1) { tscMgmtIpSet.numOfIps = 1; tscMgmtIpSet.inUse = 0; - taosGetFqdnPortFromEp(tsFirst, tscMgmtIpSet.fqdn[0], &tscMgmtIpSet.port[0]); + taosGetFqdnPortFromEp(tsFirst, tscMgmtIpSet.fqdn[0], &tscMgmtIpSet.port[0]); tscTrace("edge mgmt IP list:"); tscPrintMgmtIp(); } @@ -201,7 +201,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { } if (pSql->cmd.command < TSDB_SQL_MGMT) { - tscTrace("%p msg:%s is sent to server %d", pSql, taosMsg[pSql->cmd.msgType], pSql->ipList.port); + tscTrace("%p msg:%s is sent to server %d", pSql, taosMsg[pSql->cmd.msgType], pSql->ipList.port[0]); memcpy(pMsg, pSql->cmd.payload + tsRpcHeadSize, pSql->cmd.payloadLen); SRpcMsg rpcMsg = { @@ -329,11 +329,6 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { pRes->pRsp = NULL; } - // ignore the error information returned from mnode when set ignore flag in sql - if (pRes->code == TSDB_CODE_DB_ALREADY_EXIST && pCmd->existsCheck && pRes->rspType == TSDB_MSG_TYPE_CM_CREATE_DB_RSP) { - pRes->code = TSDB_CODE_SUCCESS; - } - /* * There is not response callback function for submit response. * The actual inserted number of points is the first number. @@ -353,7 +348,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { tscTrace("%p cmd:%d code:%s rsp len:%d", pSql, pCmd->command, tstrerror(pRes->code), pRes->rspLen); } } - + if (pRes->code == TSDB_CODE_SUCCESS && tscProcessMsgRsp[pCmd->command]) rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql); @@ -415,7 +410,6 @@ int doProcessSql(SSqlObj *pSql) { int tscProcessSql(SSqlObj *pSql) { char * name = NULL; - SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); @@ -430,42 +424,40 @@ int tscProcessSql(SSqlObj *pSql) { type = pQueryInfo->type; - // for heartbeat, numOfTables == 0; + // while numOfTables equals to 0, it must be Heartbeat assert((pQueryInfo->numOfTables == 0 && pQueryInfo->command == TSDB_SQL_HB) || pQueryInfo->numOfTables > 0); } tscTrace("%p SQL cmd:%d will be processed, name:%s, type:%d", pSql, pCmd->command, name, type); - if (pSql->cmd.command < TSDB_SQL_MGMT) { - // the pTableMetaInfo cannot be NULL + if (pCmd->command < TSDB_SQL_MGMT) { // the pTableMetaInfo cannot be NULL if (pTableMetaInfo == NULL) { pSql->res.code = TSDB_CODE_OTHERS; return pSql->res.code; } - } else if (pSql->cmd.command < TSDB_SQL_LOCAL) { - pSql->ipList = tscMgmtIpSet; + } else if (pCmd->command < TSDB_SQL_LOCAL) { + pSql->ipList = tscMgmtIpSet; //? } else { // local handler return (*tscProcessMsgRsp[pCmd->command])(pSql); } - // todo handle async situation - if (QUERY_IS_JOIN_QUERY(type)) { - if ((pQueryInfo->type & TSDB_QUERY_TYPE_SUBQUERY) == 0) { - return tscHandleMasterJoinQuery(pSql); - } else { - // for first stage sub query, iterate all vnodes to get all timestamp - if ((pQueryInfo->type & TSDB_QUERY_TYPE_JOIN_SEC_STAGE) != TSDB_QUERY_TYPE_JOIN_SEC_STAGE) { - return doProcessSql(pSql); - } - } - } - - if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { // super table query - tscHandleMasterSTableQuery(pSql); - return pRes->code; - } else if (pSql->fp == (void(*)())tscHandleMultivnodeInsert) { // multi-vnodes insertion - tscHandleMultivnodeInsert(pSql); - return pSql->res.code; - } +// if (QUERY_IS_JOIN_QUERY(type)) { +// if ((pQueryInfo->type & TSDB_QUERY_TYPE_SUBQUERY) == 0) { +// return tscHandleMasterJoinQuery(pSql); +// } else { +// // for first stage sub query, iterate all vnodes to get all timestamp +// if ((pQueryInfo->type & TSDB_QUERY_TYPE_JOIN_SEC_STAGE) != TSDB_QUERY_TYPE_JOIN_SEC_STAGE) { +// return doProcessSql(pSql); +// } +// } +// } +// +// if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { // super table query +// tscHandleMasterSTableQuery(pSql); +// return pRes->code; +// } else if (pSql->fp == (void(*)())tscHandleMultivnodeInsert) { // multi-vnodes insertion +// tscHandleMultivnodeInsert(pSql); +// return pRes->code; +// } return doProcessSql(pSql); } @@ -502,14 +494,14 @@ void tscKillSTableQuery(SSqlObj *pSql) { const int64_t MAX_WAITING_TIME = 10000; // 10 Sec. int64_t stime = taosGetTimestampMs(); - while (pSql->cmd.command != TSDB_SQL_RETRIEVE_METRIC && pSql->cmd.command != TSDB_SQL_RETRIEVE_EMPTY_RESULT) { + while (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE && pCmd->command != TSDB_SQL_RETRIEVE_EMPTY_RESULT) { taosMsleep(100); if (taosGetTimestampMs() - stime > MAX_WAITING_TIME) { break; } } - tscTrace("%p metric query is cancelled", pSql); + tscTrace("%p super table query cancelled", pSql); } int tscBuildRetrieveMsg(SSqlObj *pSql, SSqlInfo *pInfo) { @@ -570,7 +562,7 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // pSql->cmd.payloadLen is set during copying data into payload pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; - tscSetDnodeIpList(pSql, pTableMeta); + tscSetDnodeIpList(pSql, &pTableMeta->vgroupInfo); tscTrace("%p build submit msg, vgId:%d numOfVgroup:%d numberOfIP:%d", pSql, vgId, htonl(pMsgDesc->numOfVnodes), pSql->ipList.numOfIps); return TSDB_CODE_SUCCESS; @@ -600,18 +592,65 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { return size; } -static char *doSerializeTableInfo(SSqlObj *pSql, int32_t vgId, char *pMsg) { +static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char *pMsg) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - tscTrace("%p vgId:%d, query on table:%s, uid:%" PRIu64, pSql, vgId, pTableMetaInfo->name, pTableMeta->uid); + if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo) || pTableMetaInfo->pVgroupTables == NULL) { + + SCMVgroupInfo* pVgroupInfo = NULL; + if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { + pVgroupInfo = &pTableMeta->vgroupInfo; + } else { + int32_t index = pTableMetaInfo->vgroupIndex; + assert(index >= 0); + + pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index]; + tscTrace("%p query on stable, vgIndex:%d, numOfVgroups:%d", pSql, index, pTableMetaInfo->vgroupList->numOfVgroups); + } + + tscSetDnodeIpList(pSql, pVgroupInfo); + pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId); + + STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; + pTableIdInfo->tid = htonl(pTableMeta->sid); + pTableIdInfo->uid = htobe64(pTableMeta->uid); + pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid)); - STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; - pTableIdInfo->sid = htonl(pTableMeta->sid); - pTableIdInfo->uid = htobe64(pTableMeta->uid); - pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid)); + pQueryMsg->numOfTables = htonl(1); // set the number of tables + + pMsg += sizeof(STableIdInfo); + } else { + int32_t index = pTableMetaInfo->vgroupIndex; + int32_t numOfVgroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables); + assert(index >= 0 && index < numOfVgroups); + + tscTrace("%p query on stable, vgIndex:%d, numOfVgroups:%d", pSql, index, numOfVgroups); + + SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index); + + // set the vgroup info + tscSetDnodeIpList(pSql, &pTableIdList->vgInfo); + pQueryMsg->head.vgId = htonl(pTableIdList->vgInfo.vgId); + + int32_t numOfTables = taosArrayGetSize(pTableIdList->itemList); + pQueryMsg->numOfTables = htonl(numOfTables); // set the number of tables + + // serialize each table id info + for(int32_t i = 0; i < numOfTables; ++i) { + STableIdInfo* pItem = taosArrayGet(pTableIdList->itemList, i); + + STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; + pTableIdInfo->tid = htonl(pItem->tid); + pTableIdInfo->uid = htobe64(pItem->uid); +// pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid)); + pMsg += sizeof(STableIdInfo); + } + } + + tscTrace("%p vgId:%d, query on table:%s, uid:%" PRIu64, pSql, htonl(pQueryMsg->head.vgId), pTableMetaInfo->name, + pTableMeta->uid); - pMsg += sizeof(STableIdInfo); return pMsg; } @@ -629,7 +668,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - if (taosArrayGetSize(pQueryInfo->colList) <= 0) { + if (taosArrayGetSize(pQueryInfo->colList) <= 0 && !tscQueryTags(pQueryInfo)) { tscError("%p illegal value of numOfCols in query msg: %d", pSql, tscGetNumOfColumns(pTableMeta)); return -1; } @@ -648,38 +687,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pStart; - int32_t msgLen = 0; - int32_t numOfTables = 0; - int32_t numOfTags = taosArrayGetSize(pTableMetaInfo->tagColList); - - if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { - numOfTables = 1; - tscSetDnodeIpList(pSql, pTableMeta); - pQueryMsg->head.vgId = htonl(pTableMeta->vgroupInfo.vgId); - tscTrace("%p queried tables:%d, table id: %s", pSql, 1, pTableMetaInfo->name); - } else { // query super table - int32_t index = pTableMetaInfo->vgroupIndex; - if (index < 0) { - tscError("%p error vgroupIndex:%d", pSql, index); - return -1; - } - - SCMVgroupInfo* pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index]; - - pSql->ipList.numOfIps = pVgroupInfo->numOfIps; // todo fix me - pSql->ipList.inUse = 0; + int32_t numOfTags = taosArrayGetSize(pTableMetaInfo->tagColList); - for(int32_t i = 0; i < pVgroupInfo->numOfIps; ++i) { - strcpy(pSql->ipList.fqdn[i], pVgroupInfo->ipAddr[i].fqdn); - pSql->ipList.port[i] = pVgroupInfo->ipAddr[i].port; - } - - tscTrace("%p query on super table, numOfVgroup:%d, vgroupIndex:%d", pSql, pTableMetaInfo->vgroupList->numOfVgroups, index); - - pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId); - numOfTables = 1; - } - if (pQueryInfo->order.order == TSDB_ORDER_ASC) { pQueryMsg->window.skey = htobe64(pQueryInfo->window.skey); pQueryMsg->window.ekey = htobe64(pQueryInfo->window.ekey); @@ -688,7 +697,6 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->window.ekey = htobe64(pQueryInfo->window.skey); } - pQueryMsg->numOfTables = htonl(numOfTables); pQueryMsg->order = htons(pQueryInfo->order.order); pQueryMsg->orderColId = htons(pQueryInfo->order.orderColId); pQueryMsg->interpoType = htons(pQueryInfo->interpoType); @@ -793,10 +801,10 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pSqlFuncExpr = (SSqlFuncMsg *)pMsg; } - + // serialize the table info (sid, uid, tags) - pMsg = doSerializeTableInfo(pSql, htons(pQueryMsg->head.vgId), pMsg); - + pMsg = doSerializeTableInfo(pQueryMsg, pSql, pMsg); + SSqlGroupbyExpr *pGroupbyExpr = &pQueryInfo->groupbyExpr; if (pGroupbyExpr->numOfGroupCols > 0) { pQueryMsg->orderByIdx = htons(pGroupbyExpr->orderIndex); @@ -834,7 +842,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pSchema = tscGetTableTagSchema(pTableMeta); for (int32_t i = 0; i < numOfTags; ++i) { - SColumn *pCol = taosArrayGetP(pQueryInfo->colList, i); + SColumn *pCol = taosArrayGetP(pTableMetaInfo->tagColList, i); SSchema *pColSchema = &pSchema[pCol->colIndex.columnIndex]; if ((pCol->colIndex.columnIndex >= numOfTagColumns || pCol->colIndex.columnIndex < -1) || @@ -902,8 +910,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pMsg += strlen(pQueryInfo->tagCond.tbnameCond.cond) + 1; } - // tbname in/like query expression should be sent to mgmt node - msgLen = pMsg - pStart; + int32_t msgLen = pMsg - pStart; tscTrace("%p msg built success,len:%d bytes", pSql, msgLen); pCmd->payloadLen = msgLen; @@ -1459,11 +1466,11 @@ int tscProcessTagRetrieveRsp(SSqlObj *pSql) { return tscLocalResultCommonBuilder(pSql, numOfRes); } -int tscProcessRetrieveMetricRsp(SSqlObj *pSql) { +int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; - pRes->code = tscDoLocalreduce(pSql); + pRes->code = tscDoLocalMerge(pSql); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); if (pRes->code == TSDB_CODE_SUCCESS && pRes->numOfRows > 0) { @@ -1473,12 +1480,10 @@ int tscProcessRetrieveMetricRsp(SSqlObj *pSql) { pRes->row = 0; uint8_t code = pRes->code; - if (pSql->fp) { // async retrieve metric data - if (pRes->code == TSDB_CODE_SUCCESS) { - (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); - } else { - tscQueueAsyncRes(pSql); - } + if (pRes->code == TSDB_CODE_SUCCESS) { + (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); + } else { + tscQueueAsyncRes(pSql); } return code; @@ -1590,32 +1595,32 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { return pCmd->payloadLen; } -static UNUSED_FUNC int32_t tscEstimateMetricMetaMsgSize(SSqlCmd *pCmd) { - const int32_t defaultSize = - minMsgSize() + sizeof(SSuperTableMetaMsg) + sizeof(SMgmtHead) + sizeof(int16_t) * TSDB_MAX_TAGS; - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - - int32_t n = 0; - size_t size = taosArrayGetSize(pQueryInfo->tagCond.pCond); - for (int32_t i = 0; i < size; ++i) { - assert(0); -// n += strlen(pQueryInfo->tagCond.cond[i].cond); - } - - int32_t tagLen = n * TSDB_NCHAR_SIZE; - if (pQueryInfo->tagCond.tbnameCond.cond != NULL) { - tagLen += strlen(pQueryInfo->tagCond.tbnameCond.cond) * TSDB_NCHAR_SIZE; - } - - int32_t joinCondLen = (TSDB_TABLE_ID_LEN + sizeof(int16_t)) * 2; - int32_t elemSize = sizeof(SSuperTableMetaElemMsg) * pQueryInfo->numOfTables; - - int32_t colSize = pQueryInfo->groupbyExpr.numOfGroupCols*sizeof(SColIndex); - - int32_t len = tagLen + joinCondLen + elemSize + colSize + defaultSize; - - return MAX(len, TSDB_DEFAULT_PAYLOAD_SIZE); -} +//static UNUSED_FUNC int32_t tscEstimateMetricMetaMsgSize(SSqlCmd *pCmd) { +//// const int32_t defaultSize = +//// minMsgSize() + sizeof(SSuperTableMetaMsg) + sizeof(SMgmtHead) + sizeof(int16_t) * TSDB_MAX_TAGS; +//// SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); +//// +//// int32_t n = 0; +//// size_t size = taosArrayGetSize(pQueryInfo->tagCond.pCond); +//// for (int32_t i = 0; i < size; ++i) { +//// assert(0); +////// n += strlen(pQueryInfo->tagCond.cond[i].cond); +//// } +//// +//// int32_t tagLen = n * TSDB_NCHAR_SIZE; +//// if (pQueryInfo->tagCond.tbnameCond.cond != NULL) { +//// tagLen += strlen(pQueryInfo->tagCond.tbnameCond.cond) * TSDB_NCHAR_SIZE; +//// } +//// +//// int32_t joinCondLen = (TSDB_TABLE_ID_LEN + sizeof(int16_t)) * 2; +//// int32_t elemSize = sizeof(SSuperTableMetaElemMsg) * pQueryInfo->numOfTables; +//// +//// int32_t colSize = pQueryInfo->groupbyExpr.numOfGroupCols*sizeof(SColIndex); +//// +//// int32_t len = tagLen + joinCondLen + elemSize + colSize + defaultSize; +//// +//// return MAX(len, TSDB_DEFAULT_PAYLOAD_SIZE); +//} int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) { @@ -1763,14 +1768,22 @@ int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) { #endif SSqlCmd *pCmd = &pSql->cmd; - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - - SCMSTableVgroupMsg *pStableVgroupMsg = (SCMSTableVgroupMsg *) pCmd->payload; - strncpy(pStableVgroupMsg->tableId, pTableMetaInfo->name, tListLen(pStableVgroupMsg->tableId)); + + char* pMsg = pCmd->payload; + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + + SCMSTableVgroupMsg *pStableVgroupMsg = (SCMSTableVgroupMsg *) pMsg; + pStableVgroupMsg->numOfTables = htonl(pQueryInfo->numOfTables); + pMsg += sizeof(SCMSTableVgroupMsg); + + for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, i); + strncpy(pMsg, pTableMetaInfo->name, TSDB_TABLE_ID_LEN); + pMsg += TSDB_TABLE_ID_LEN; + } pCmd->msgType = TSDB_MSG_TYPE_CM_STABLE_VGROUP; - pCmd->payloadLen = sizeof(SCMSTableVgroupMsg); + pCmd->payloadLen = (pMsg - pCmd->payload); return TSDB_CODE_SUCCESS; } @@ -2125,27 +2138,37 @@ _error_clean: #endif SSqlRes* pRes = &pSql->res; + // NOTE: the order of several table must be preserved. SCMSTableVgroupRspMsg *pStableVgroup = (SCMSTableVgroupRspMsg *)pRes->pRsp; - pStableVgroup->numOfVgroups = htonl(pStableVgroup->numOfVgroups); + pStableVgroup->numOfTables = htonl(pStableVgroup->numOfTables); + char* pMsg = pRes->pRsp + sizeof(SCMSTableVgroupRspMsg); // master sqlObj locates in param SSqlObj* parent = pSql->param; assert(parent != NULL); SSqlCmd* pCmd = &parent->cmd; - STableMetaInfo* pInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - - pInfo->vgroupList = malloc(pRes->rspLen); - memcpy(pInfo->vgroupList, pStableVgroup, pRes->rspLen); - - for(int32_t i = 0; i < pInfo->vgroupList->numOfVgroups; ++i) { - SCMVgroupInfo* pVgroups = &pInfo->vgroupList->vgroups[i]; - - pVgroups->vgId = htonl(pVgroups->vgId); - assert(pVgroups->numOfIps >= 1); - - for(int32_t j = 0; j < pVgroups->numOfIps; ++j) { - pVgroups->ipAddr[j].port = htons(pVgroups->ipAddr[j].port); + for(int32_t i = 0; i < pStableVgroup->numOfTables; ++i) { + STableMetaInfo *pInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, i); + SVgroupsInfo * pVgroupInfo = (SVgroupsInfo *)pMsg; + pVgroupInfo->numOfVgroups = htonl(pVgroupInfo->numOfVgroups); + + size_t size = sizeof(SCMVgroupInfo) * pVgroupInfo->numOfVgroups + sizeof(SVgroupsInfo); + pInfo->vgroupList = calloc(1, size); + assert(pInfo->vgroupList != NULL); + + memcpy(pInfo->vgroupList, pVgroupInfo, size); + for (int32_t j = 0; j < pInfo->vgroupList->numOfVgroups; ++j) { + SCMVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j]; + + pVgroups->vgId = htonl(pVgroups->vgId); + assert(pVgroups->numOfIps >= 1); + + for (int32_t k = 0; k < pVgroups->numOfIps; ++k) { + pVgroups->ipAddr[k].port = htons(pVgroups->ipAddr[k].port); + } + + pMsg += size; } } @@ -2239,7 +2262,7 @@ int tscProcessConnectRsp(SSqlObj *pSql) { strcpy(pObj->sversion, pConnect->serverVersion); pObj->writeAuth = pConnect->writeAuth; pObj->superAuth = pConnect->superAuth; - taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, pObj, tscTmr, &pObj->pTimer); +// taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, pObj, tscTmr, &pObj->pTimer); return 0; } @@ -2278,7 +2301,6 @@ int tscProcessDropTableRsp(SSqlObj *pSql) { if (pTableMetaInfo->pTableMeta) { taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true); -// taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), true); } return 0; @@ -2504,42 +2526,27 @@ int tscRenewMeterMeta(SSqlObj *pSql, char *tableId) { return code; } -int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { - int code = TSDB_CODE_NETWORK_UNAVAIL; - SSqlCmd *pCmd = &pSql->cmd; - - //the query condition is serialized into pCmd->payload, we need to rebuild key for stable meta info in cache. -// bool required = false; - +static bool allVgroupInfoRetrieved(SSqlCmd* pCmd, int32_t clauseIndex) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); - if (pQueryInfo->pTableMetaInfo[0]->vgroupList != NULL) { - return TSDB_CODE_SUCCESS; - } - -#if 0 for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - char tagstr[TSDB_MAX_TAGS_LEN + 1] = {0}; - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - tscGetMetricMetaCacheKey(pQueryInfo, tagstr, pTableMetaInfo->pTableMeta->uid); - -// taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), false); - - SSuperTableMeta *ppMeta = (SSuperTableMeta *)taosCacheAcquireByName(tscCacheHandle, tagstr); - if (ppMeta == NULL) { - required = true; - break; - } else { -// pTableMetaInfo->pMetricMeta = ppMeta; + if (pTableMetaInfo->vgroupList == NULL) { + return false; } } + + // all super tables vgroupinfo are retrieved, no need to retrieve vgroup info anymore + return true; +} - // all metricmeta for one clause are retrieved from cache, no need to retrieve metricmeta from management node - if (!required) { +int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { + int code = TSDB_CODE_NETWORK_UNAVAIL; + SSqlCmd *pCmd = &pSql->cmd; + + if (allVgroupInfoRetrieved(pCmd, clauseIndex)) { return TSDB_CODE_SUCCESS; } -#endif - + SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); pNew->pTscObj = pSql->pTscObj; pNew->signature = pNew; @@ -2551,11 +2558,11 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { return code; } + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo *pMMInfo = tscGetMetaInfo(pQueryInfo, i); - - STableMeta *pTableMeta = taosCacheAcquireByName(tscCacheHandle, pMMInfo->name); - tscAddTableMetaInfo(pNewQueryInfo, pMMInfo->name, pTableMeta, NULL, pMMInfo->tagColList); + STableMetaInfo *pMInfo = tscGetMetaInfo(pQueryInfo, i); + STableMeta *pTableMeta = taosCacheAcquireByData(tscCacheHandle, pMInfo->pTableMeta); + tscAddTableMetaInfo(pNewQueryInfo, pMInfo->name, pTableMeta, NULL, pMInfo->tagColList); } if ((code = tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { @@ -2563,25 +2570,9 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { return code; } - tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond); - - pNewQueryInfo->groupbyExpr = pQueryInfo->groupbyExpr; pNewQueryInfo->numOfTables = pQueryInfo->numOfTables; + tscTrace("%p new sqlObj:%p to get vgroupInfo, numOfTables:%d", pSql, pNew, pNewQueryInfo->numOfTables); - pNewQueryInfo->slimit = pQueryInfo->slimit; - pNewQueryInfo->order = pQueryInfo->order; - - STagCond* pTagCond = &pNewQueryInfo->tagCond; - tscTrace("%p new sqlobj:%p info, numOfTables:%d, slimit:%" PRId64 ", soffset:%" PRId64 ", order:%d, tbname cond:%s", - pSql, pNew, pNewQueryInfo->numOfTables, pNewQueryInfo->slimit.limit, pNewQueryInfo->slimit.offset, - pNewQueryInfo->order.order, pTagCond->tbnameCond.cond) - -// if (pSql->fp != NULL && pSql->pStream == NULL) { -// pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); -// tscFreeQueryInfo(pCmd); -// } - - tscTrace("%p allocate new pSqlObj:%p to get stable vgroupInfo", pSql, pNew); pNew->fp = tscTableMetaCallBack; pNew->param = pSql; code = tscProcessSql(pNew); @@ -2643,7 +2634,6 @@ void tscInitMsgsFp() { tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. tscProcessMsgRsp[TSDB_SQL_DESCRIBE_TABLE] = tscProcessDescribeTableRsp; - tscProcessMsgRsp[TSDB_SQL_RETRIEVE_TAGS] = tscProcessTagRetrieveRsp; tscProcessMsgRsp[TSDB_SQL_CURRENT_DB] = tscProcessTagRetrieveRsp; tscProcessMsgRsp[TSDB_SQL_CURRENT_USER] = tscProcessTagRetrieveRsp; tscProcessMsgRsp[TSDB_SQL_SERV_VERSION] = tscProcessTagRetrieveRsp; @@ -2652,7 +2642,7 @@ void tscInitMsgsFp() { tscProcessMsgRsp[TSDB_SQL_RETRIEVE_EMPTY_RESULT] = tscProcessEmptyResultRsp; - tscProcessMsgRsp[TSDB_SQL_RETRIEVE_METRIC] = tscProcessRetrieveMetricRsp; + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_LOCALMERGE] = tscProcessRetrieveLocalMergeRsp; tscProcessMsgRsp[TSDB_SQL_ALTER_TABLE] = tscProcessAlterTableMsgRsp; tscProcessMsgRsp[TSDB_SQL_ALTER_DB] = tscProcessAlterDbMsgRsp; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 1934c05014194deef8a8427615adf2855e5343cc..aff4ad525b4c203010adebe5422427b89da28a5c 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -220,8 +220,9 @@ TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void taos_close(TAOS *taos) { STscObj *pObj = (STscObj *)taos; - if (pObj == NULL) return; - if (pObj->signature != pObj) return; + if (pObj == NULL || pObj->signature != pObj) { + return; + } if (pObj->pHb != NULL) { tscSetFreeHeatBeat(pObj); @@ -266,7 +267,7 @@ int taos_query_imp(STscObj *pObj, SSqlObj *pSql) { } if (pRes->code != TSDB_CODE_SUCCESS) { - tscFreeSqlObjPartial(pSql); + tscPartiallyFreeSqlObj(pSql); } return pRes->code; @@ -414,7 +415,7 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) { } // secondary merge has handle this situation - if (pCmd->command != TSDB_SQL_RETRIEVE_METRIC) { + if (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE) { pRes->numOfTotalInCurrentClause += pRes->numOfRows; } @@ -422,8 +423,9 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) { if (pQueryInfo == NULL) return 0; + assert(0); for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - pRes->tsrow[i] = tscGetResultColumnChr(pRes, pQueryInfo, i); + pRes->tsrow[i] = tscGetResultColumnChr(pRes, pQueryInfo, i, 0); } *rows = pRes->tsrow; @@ -431,32 +433,9 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) { return (pQueryInfo->order.order == TSDB_ORDER_DESC) ? pRes->numOfRows : -pRes->numOfRows; } -static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pField) { - SSqlRes *pRes = &pSql->res; - - if (isNull(pRes->tsrow[columnIndex], pField->type)) { - pRes->tsrow[columnIndex] = NULL; - } else if (pField->type == TSDB_DATA_TYPE_NCHAR) { - // convert unicode to native code in a temporary buffer extra one byte for terminated symbol - if (pRes->buffer[columnIndex] == NULL) { - pRes->buffer[columnIndex] = malloc(pField->bytes + TSDB_NCHAR_SIZE); - } - - /* string terminated char for binary data*/ - memset(pRes->buffer[columnIndex], 0, pField->bytes + TSDB_NCHAR_SIZE); - - if (taosUcs4ToMbs(pRes->tsrow[columnIndex], pField->bytes, pRes->buffer[columnIndex])) { - pRes->tsrow[columnIndex] = pRes->buffer[columnIndex]; - } else { - tscError("%p charset:%s to %s. val:%ls convert failed.", pSql, DEFAULT_UNICODE_ENCODEC, tsCharset, pRes->tsrow); - pRes->tsrow[columnIndex] = NULL; - } - } -} - -static char *getArithemicInputSrc(void *param, const char *name, int32_t colId) { +static UNUSED_FUNC char *getArithemicInputSrc(void *param, const char *name, int32_t colId) { // SArithmeticSupport *pSupport = (SArithmeticSupport *)param; -// SArithExprInfo * pExpr = pSupport->pArithExpr; +// SExprInfo * pExpr = pSupport->pArithExpr; // int32_t index = -1; // for (int32_t i = 0; i < pExpr->numOfCols; ++i) { @@ -471,210 +450,6 @@ static char *getArithemicInputSrc(void *param, const char *name, int32_t colId) return 0; } -static void **doSetResultRowData(SSqlObj *pSql) { - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - assert(pRes->row >= 0 && pRes->row <= pRes->numOfRows); - - if (pRes->row >= pRes->numOfRows) { // all the results has returned to invoker - tfree(pRes->tsrow); - return pRes->tsrow; - } - - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - - //todo refactor move away - size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); - for(int32_t k = 0; k < numOfExprs; ++k) { - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, k); - - if (k > 0) { - SSqlExpr* pPrev = tscSqlExprGet(pQueryInfo, k - 1); - pExpr->offset = pPrev->offset + pPrev->resBytes; - } - } - - int32_t num = 0; - for (int i = 0; i < tscNumOfFields(pQueryInfo); ++i) { - SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, i); - if (pInfo->pSqlExpr != NULL) { - pRes->tsrow[i] = tscGetResultColumnChr(pRes, pQueryInfo, i) + pInfo->pSqlExpr->resBytes * pRes->row; - } else { - assert(0); - } - - // primary key column cannot be null in interval query, no need to check - if (i == 0 && pQueryInfo->intervalTime > 0) { - continue; - } - - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - transferNcharData(pSql, i, pField); - - // calculate the result from several other columns - if (pInfo->pArithExprInfo != NULL) { - SArithmeticSupport *sas = (SArithmeticSupport *)calloc(1, sizeof(SArithmeticSupport)); - sas->offset = 0; - sas->pArithExpr = pInfo->pArithExprInfo; - -// sas->numOfCols = sas->pArithExpr->numOfCols; - - if (pRes->buffer[i] == NULL) { - pRes->buffer[i] = malloc(tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i)->bytes); - } - - for(int32_t k = 0; k < sas->numOfCols; ++k) { -// int32_t columnIndex = sas->pArithExpr->colList[k].colIndex; -// SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, columnIndex); -// -// sas->elemSize[k] = pExpr->resBytes; -// sas->data[k] = (pRes->data + pRes->numOfRows* pExpr->offset) + pRes->row*pExpr->resBytes; - } - - tExprTreeCalcTraverse(sas->pArithExpr->pExpr, 1, pRes->buffer[i], sas, TSDB_ORDER_ASC, getArithemicInputSrc); - pRes->tsrow[i] = pRes->buffer[i]; - - free(sas); //todo optimization - } - } - - assert(num <= pQueryInfo->fieldsInfo.numOfOutput); - - pRes->row++; // index increase one-step - return pRes->tsrow; -} - -static bool tscHashRemainDataInSubqueryResultSet(SSqlObj *pSql) { - bool hasData = true; - SSqlCmd *pCmd = &pSql->cmd; - - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { - bool allSubqueryExhausted = true; - - for (int32_t i = 0; i < pSql->numOfSubs; ++i) { - if (pSql->pSubs[i] == NULL) { - continue; - } - -// SSqlRes *pRes1 = &pSql->pSubs[i]->res; - SSqlCmd *pCmd1 = &pSql->pSubs[i]->cmd; - - SQueryInfo * pQueryInfo1 = tscGetQueryInfoDetail(pCmd1, pCmd1->clauseIndex); -// STableMetaInfo *pMetaInfo = tscGetMetaInfo(pQueryInfo1, 0); - - assert(pQueryInfo1->numOfTables == 1); - - /* - * if the global limitation is not reached, and current result has not exhausted, or next more vnodes are - * available, goes on - */ -// if (pMetaInfo->vnodeIndex < pMetaInfo->pMetricMeta->numOfVnodes && pRes1->row < pRes1->numOfRows && -// (!tscHasReachLimitation(pQueryInfo1, pRes1))) { -// allSubqueryExhausted = false; -// break; -// } - } - - hasData = !allSubqueryExhausted; - } else { // otherwise, in case inner join, if any subquery exhausted, query completed. - for (int32_t i = 0; i < pSql->numOfSubs; ++i) { - if (pSql->pSubs[i] == 0) { - continue; - } - - SSqlRes * pRes1 = &pSql->pSubs[i]->res; - SQueryInfo *pQueryInfo1 = tscGetQueryInfoDetail(&pSql->pSubs[i]->cmd, 0); - - if ((pRes1->row >= pRes1->numOfRows && tscHasReachLimitation(pQueryInfo1, pRes1) && - tscProjectionQueryOnTable(pQueryInfo1)) || - (pRes1->numOfRows == 0)) { - hasData = false; - break; - } - } - } - - return hasData; -} - -static UNUSED_FUNC void **tscBuildResFromSubqueries(SSqlObj *pSql) { - SSqlRes *pRes = &pSql->res; - - while (1) { - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); - size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); - - if (pRes->tsrow == NULL) { - pRes->tsrow = calloc(numOfExprs, POINTER_BYTES); - } - - bool success = false; - - int32_t numOfTableHasRes = 0; - for (int32_t i = 0; i < pSql->numOfSubs; ++i) { - if (pSql->pSubs[i] != 0) { - numOfTableHasRes++; - } - } - - if (numOfTableHasRes >= 2) { // do merge result - success = (doSetResultRowData(pSql->pSubs[0]) != NULL) && (doSetResultRowData(pSql->pSubs[1]) != NULL); - } else { // only one subquery - SSqlObj *pSub = pSql->pSubs[0]; - if (pSub == NULL) { - pSub = pSql->pSubs[1]; - } - - success = (doSetResultRowData(pSub) != NULL); - } - - if (success) { // current row of final output has been built, return to app - for (int32_t i = 0; i < numOfExprs; ++i) { - int32_t tableIndex = pRes->pColumnIndex[i].tableIndex; - int32_t columnIndex = pRes->pColumnIndex[i].columnIndex; - - SSqlRes *pRes1 = &pSql->pSubs[tableIndex]->res; - pRes->tsrow[i] = pRes1->tsrow[columnIndex]; - } - - pRes->numOfTotalInCurrentClause++; - - break; - } else { // continue retrieve data from vnode - if (!tscHashRemainDataInSubqueryResultSet(pSql)) { - tscTrace("%p at least one subquery exhausted, free all other %d subqueries", pSql, pSql->numOfSubs - 1); - SSubqueryState *pState = NULL; - - // free all sub sqlobj - for (int32_t i = 0; i < pSql->numOfSubs; ++i) { - SSqlObj *pChildObj = pSql->pSubs[i]; - if (pChildObj == NULL) { - continue; - } - - SJoinSubquerySupporter *pSupporter = (SJoinSubquerySupporter *)pChildObj->param; - pState = pSupporter->pState; - - tscDestroyJoinSupporter(pChildObj->param); - taos_free_result(pChildObj); - } - - free(pState); - return NULL; - } - - tscFetchDatablockFromSubquery(pSql); - if (pRes->code != TSDB_CODE_SUCCESS) { - return NULL; - } - } - } - - return pRes->tsrow; -} - static void waitForRetrieveRsp(void *param, TAOS_RES *tres, int numOfRows) { SSqlObj* pSql = (SSqlObj*) tres; @@ -701,15 +476,19 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { } // current data are exhausted, fetch more data - if (pRes->data == NULL || (pRes->data != NULL && pRes->row >= pRes->numOfRows && pRes->completed != true && - (pCmd->command == TSDB_SQL_RETRIEVE || pCmd->command == TSDB_SQL_RETRIEVE_METRIC || - pCmd->command == TSDB_SQL_FETCH || pCmd->command == TSDB_SQL_DESCRIBE_TABLE))) { + if (pRes->row >= pRes->numOfRows && pRes->completed != true && + (pCmd->command == TSDB_SQL_RETRIEVE || + pCmd->command == TSDB_SQL_RETRIEVE_LOCALMERGE || + pCmd->command == TSDB_SQL_METRIC_JOIN_RETRIEVE || + pCmd->command == TSDB_SQL_FETCH || + pCmd->command == TSDB_SQL_SHOW || + pCmd->command == TSDB_SQL_SELECT || + pCmd->command == TSDB_SQL_DESCRIBE_TABLE)) { taos_fetch_rows_a(res, waitForRetrieveRsp, pSql->pTscObj); - sem_wait(&pSql->rspSem); } - return doSetResultRowData(pSql); + return doSetResultRowData(pSql, true); } int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { @@ -795,7 +574,7 @@ void taos_free_result_imp(TAOS_RES *res, int keepCmd) { if (keepCmd) { tscFreeSqlResult(pSql); } else { - tscFreeSqlObjPartial(pSql); + tscPartiallyFreeSqlObj(pSql); } } @@ -805,7 +584,7 @@ void taos_free_result_imp(TAOS_RES *res, int keepCmd) { // set freeFlag to 1 in retrieve message if there are un-retrieved results SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); if (pQueryInfo == NULL) { - tscFreeSqlObjPartial(pSql); + tscPartiallyFreeSqlObj(pSql); return; } @@ -857,7 +636,7 @@ void taos_free_result_imp(TAOS_RES *res, int keepCmd) { tscFreeSqlResult(pSql); tscTrace("%p sql result is freed by app while sql command is kept", pSql); } else { - tscFreeSqlObjPartial(pSql); + tscPartiallyFreeSqlObj(pSql); tscTrace("%p sql result is freed by app", pSql); } } else { // for async release, remove its link @@ -878,7 +657,7 @@ void taos_free_result_imp(TAOS_RES *res, int keepCmd) { tscFreeSqlResult(pSql); tscTrace("%p sql result is freed while sql command is kept", pSql); } else { - tscFreeSqlObjPartial(pSql); + tscPartiallyFreeSqlObj(pSql); tscTrace("%p sql result is freed by app", pSql); } } @@ -1223,7 +1002,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { tscTrace("%p load multi metermeta result:%d %s pObj:%p", pSql, pRes->code, taos_errstr(taos), pObj); if (pRes->code != TSDB_CODE_SUCCESS) { - tscFreeSqlObjPartial(pSql); + tscPartiallyFreeSqlObj(pSql); } return pRes->code; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 68029ac71928dd5744f0a9cffc6dcd6cf4351bb5..9aceecd377cd682442a63f6a6f51826dca723e74 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -14,19 +14,22 @@ */ #include "tscSubquery.h" +#include +#include #include "os.h" #include "qtsbuf.h" -#include "tsclient.h" #include "tscLog.h" +#include "tsclient.h" typedef struct SInsertSupporter { SSubqueryState* pState; SSqlObj* pSql; } SInsertSupporter; -static void freeSubqueryObj(SSqlObj* pSql); +static void freeJoinSubqueryObj(SSqlObj* pSql); +static bool tscHashRemainDataInSubqueryResultSet(SSqlObj *pSql); -static bool doCompare(int32_t order, int64_t left, int64_t right) { +static bool tsCompare(int32_t order, int64_t left, int64_t right) { if (order == TSDB_ORDER_ASC) { return left < right; } else { @@ -34,8 +37,8 @@ static bool doCompare(int32_t order, int64_t left, int64_t right) { } } -static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSubquerySupporter* pSupporter1, - SJoinSubquerySupporter* pSupporter2, TSKEY* st, TSKEY* et) { +static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, + SJoinSupporter* pSupporter2, TSKEY* st, TSKEY* et) { STSBuf* output1 = tsBufCreate(true); STSBuf* output2 = tsBufCreate(true); @@ -81,17 +84,16 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSubquerySupporter* pSuppor STSElem elem2 = tsBufGetElem(pSupporter2->pTSBuf); #ifdef _DEBUG_VIEW - // for debug purpose tscPrint("%" PRId64 ", tags:%d \t %" PRId64 ", tags:%d", elem1.ts, elem1.tag, elem2.ts, elem2.tag); #endif - if (elem1.tag < elem2.tag || (elem1.tag == elem2.tag && doCompare(order, elem1.ts, elem2.ts))) { + if (elem1.tag < elem2.tag || (elem1.tag == elem2.tag && tsCompare(order, elem1.ts, elem2.ts))) { if (!tsBufNextPos(pSupporter1->pTSBuf)) { break; } numOfInput1++; - } else if (elem1.tag > elem2.tag || (elem1.tag == elem2.tag && doCompare(order, elem2.ts, elem1.ts))) { + } else if (elem1.tag > elem2.tag || (elem1.tag == elem2.tag && tsCompare(order, elem2.ts, elem1.ts))) { if (!tsBufNextPos(pSupporter2->pTSBuf)) { break; } @@ -155,8 +157,8 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSubquerySupporter* pSuppor } // todo handle failed to create sub query -SJoinSubquerySupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, int32_t index) { - SJoinSubquerySupporter* pSupporter = calloc(1, sizeof(SJoinSubquerySupporter)); +SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, int32_t index) { + SJoinSupporter* pSupporter = calloc(1, sizeof(SJoinSupporter)); if (pSupporter == NULL) { return NULL; } @@ -172,7 +174,6 @@ SJoinSubquerySupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pS STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, index); pSupporter->uid = pTableMetaInfo->pTableMeta->uid; - assert (pSupporter->uid != 0); getTmpfilePath("join-", pSupporter->path); @@ -185,19 +186,25 @@ SJoinSubquerySupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pS return pSupporter; } -void tscDestroyJoinSupporter(SJoinSubquerySupporter* pSupporter) { +void tscDestroyJoinSupporter(SJoinSupporter* pSupporter) { if (pSupporter == NULL) { return; } - tscSqlExprInfoDestroy(pSupporter->exprsInfo); - tscColumnListDestroy(pSupporter->colList); + if (pSupporter->exprList != NULL) { + tscSqlExprInfoDestroy(pSupporter->exprList); + } + + if (pSupporter->colList != NULL) { + tscColumnListDestroy(pSupporter->colList); + } tscFieldInfoClear(&pSupporter->fieldsInfo); if (pSupporter->f != NULL) { fclose(pSupporter->f); unlink(pSupporter->path); + pSupporter->f = NULL; } tscTagCondRelease(&pSupporter->tagCond); @@ -227,18 +234,16 @@ bool needSecondaryQuery(SQueryInfo* pQueryInfo) { * launch secondary stage query to fetch the result that contains timestamp in set */ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { - int32_t numOfSub = 0; - SJoinSubquerySupporter* pSupporter = NULL; + int32_t numOfSub = 0; + SJoinSupporter* pSupporter = NULL; /* - * If the columns are not involved in the final select clause, the secondary query will not be launched - * for the subquery. + * If the columns are not involved in the final select clause, + * the corresponding query will not be issued. */ - SSubqueryState* pState = NULL; - for (int32_t i = 0; i < pSql->numOfSubs; ++i) { pSupporter = pSql->pSubs[i]->param; - if (taosArrayGetSize(pSupporter->exprsInfo) > 0) { + if (taosArrayGetSize(pSupporter->exprList) > 0) { ++numOfSub; } } @@ -249,10 +254,8 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { tscTrace("%p start to launch secondary subqueries, total:%d, only:%d needs to query, others are not retrieve in " "select clause", pSql, pSql->numOfSubs, numOfSub); - /* - * the subqueries that do not actually launch the secondary query to virtual node is set as completed. - */ - pState = pSupporter->pState; + //the subqueries that do not actually launch the secondary query to virtual node is set as completed. + SSubqueryState* pState = pSupporter->pState; pState->numOfTotal = pSql->numOfSubs; pState->numOfCompleted = (pSql->numOfSubs - numOfSub); @@ -264,8 +267,8 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { pSupporter = pPrevSub->param; - if (taosArrayGetSize(pSupporter->exprsInfo) == 0) { - tscTrace("%p subIndex: %d, not need to launch query, ignore it", pSql, i); + if (taosArrayGetSize(pSupporter->exprList) == 0) { + tscTrace("%p subIndex: %d, no need to launch query, ignore it", pSql, i); tscDestroyJoinSupporter(pSupporter); tscFreeSqlObj(pPrevSub); @@ -296,27 +299,20 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { pQueryInfo->tsBuf = pTSBuf; // transfer the ownership of timestamp comp-z data to the new created object // set the second stage sub query for join process - pQueryInfo->type |= TSDB_QUERY_TYPE_JOIN_SEC_STAGE; + TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE); pQueryInfo->intervalTime = pSupporter->interval; pQueryInfo->groupbyExpr = pSupporter->groupbyExpr; - - tscColumnListCopy(pQueryInfo->colList, pSupporter->colList, 0); + tscTagCondCopy(&pQueryInfo->tagCond, &pSupporter->tagCond); - pQueryInfo->exprsInfo = tscSqlExprCopy(pSupporter->exprsInfo, pSupporter->uid, false); - tscFieldInfoCopy(&pQueryInfo->fieldsInfo, &pSupporter->fieldsInfo); + pQueryInfo->colList = pSupporter->colList; + pQueryInfo->exprList = pSupporter->exprList; + pQueryInfo->fieldsInfo = pSupporter->fieldsInfo; - pSupporter->fieldsInfo.numOfOutput = 0; - - /* - * if the first column of the secondary query is not ts function, add this function. - * Because this column is required to filter with timestamp after intersecting. - */ - SSqlExpr* pExpr = taosArrayGet(pSupporter->exprsInfo, 0); - if (pExpr->functionId != TSDB_FUNC_TS) { - tscAddTimestampColumn(pQueryInfo, TSDB_FUNC_TS, 0); - } + pSupporter->exprList = NULL; + pSupporter->colList = NULL; + memset(&pSupporter->fieldsInfo, 0, sizeof(SFieldInfo)); SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); assert(pNew->numOfSubs == 0 && pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1); @@ -334,7 +330,7 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { // fetch the join tag column if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { - SSqlExpr *pExpr = tscSqlExprGet(pNewQueryInfo, 0); + SSqlExpr* pExpr = tscSqlExprGet(pNewQueryInfo, 0); assert(pQueryInfo->tagCond.joinInfo.hasJoin); int16_t tagColIndex = tscGetJoinTagColIndexByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid); @@ -342,12 +338,10 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { pExpr->numOfParams = 1; } - tscPrintSelectClause(pNew, 0); - size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList); tscTrace("%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s", pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, - taosArrayGetSize(pNewQueryInfo->exprsInfo), numOfCols, + taosArrayGetSize(pNewQueryInfo->exprList), numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, pNewQueryInfo->pTableMetaInfo[0]->name); } @@ -356,7 +350,7 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { pSql->res.code = TSDB_CODE_CLI_OUT_OF_MEMORY; tscError("%p failed to prepare subqueries objs for secondary phase query, numOfSub:%d, code:%d", pSql, pSql->numOfSubs, pSql->res.code); - freeSubqueryObj(pSql); + freeJoinSubqueryObj(pSql); return pSql->res.code; } @@ -373,19 +367,22 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { return TSDB_CODE_SUCCESS; } -void freeSubqueryObj(SSqlObj* pSql) { +void freeJoinSubqueryObj(SSqlObj* pSql) { SSubqueryState* pState = NULL; for (int32_t i = 0; i < pSql->numOfSubs; ++i) { - if (pSql->pSubs[i] != NULL) { - SJoinSubquerySupporter* p = pSql->pSubs[i]->param; - pState = p->pState; + SSqlObj* pSub = pSql->pSubs[i]; + if (pSub == NULL) { + continue; + } + + SJoinSupporter* p = pSub->param; + pState = p->pState; - tscDestroyJoinSupporter(p); + tscDestroyJoinSupporter(p); - if (pSql->pSubs[i]->res.code == TSDB_CODE_SUCCESS) { - taos_free_result(pSql->pSubs[i]); - } + if (pSub->res.code == TSDB_CODE_SUCCESS) { + taos_free_result(pSub); } } @@ -393,24 +390,14 @@ void freeSubqueryObj(SSqlObj* pSql) { pSql->numOfSubs = 0; } -static void doQuitSubquery(SSqlObj* pParentSql) { - freeSubqueryObj(pParentSql); - -// tsem_wait(&pParentSql->emptyRspSem); -// tsem_wait(&pParentSql->emptyRspSem); - -// tsem_post(&pParentSql->rspSem); -} - -static void quitAllSubquery(SSqlObj* pSqlObj, SJoinSubquerySupporter* pSupporter) { +static void quitAllSubquery(SSqlObj* pSqlObj, SJoinSupporter* pSupporter) { int32_t numOfTotal = pSupporter->pState->numOfTotal; int32_t finished = atomic_add_fetch_32(&pSupporter->pState->numOfCompleted, 1); + pSqlObj->res.code = pSupporter->pState->code; if (finished >= numOfTotal) { - pSqlObj->res.code = abs(pSupporter->pState->code); tscError("%p all subquery return and query failed, global code:%d", pSqlObj, pSqlObj->res.code); - - doQuitSubquery(pSqlObj); + freeJoinSubqueryObj(pSqlObj); } } @@ -422,119 +409,368 @@ static void updateQueryTimeRange(SQueryInfo* pQueryInfo, int64_t st, int64_t et) pQueryInfo->window.ekey = et; } -static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { - SJoinSubquerySupporter* pSupporter = (SJoinSubquerySupporter*)param; - SSqlObj* pParentSql = pSupporter->pObj; - - SSqlObj* pSql = (SSqlObj*)tres; - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; +static void tSIntersectionAndLaunchSecQuery(SJoinSupporter* pSupporter, SSqlObj* pSql) { + SSqlObj* pParentSql = pSupporter->pObj; +// SSqlCmd* pCmd = &pSql->cmd; +// SSqlRes* pRes = &pSql->res; - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); +// SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + SQueryInfo* pParentQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, pParentSql->cmd.clauseIndex); + +// if (tscNonOrderedProjectionQueryOnSTable(pParentQueryInfo, 0)) { +// STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); +// assert(pQueryInfo->numOfTables == 1); +// +// // for projection query, need to try next vnode +//// int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes; +// int32_t totalVnode = 0; +// if ((++pTableMetaInfo->vgroupIndex) < totalVnode) { +// tscTrace("%p current vnode:%d exhausted, try next:%d. total vnode:%d. current numOfRes:%d", pSql, +// pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVnode, pRes->numOfTotal); +// +// pSql->cmd.command = TSDB_SQL_SELECT; +// pSql->fp = tscJoinQueryCallback; +// tscProcessSql(pSql); +// +// return; +// } +// } + + int32_t numOfTotal = pSupporter->pState->numOfTotal; + int32_t finished = atomic_add_fetch_32(&pSupporter->pState->numOfCompleted, 1); - if ((pQueryInfo->type & TSDB_QUERY_TYPE_JOIN_SEC_STAGE) == 0) { + if (finished >= numOfTotal) { + assert(finished == numOfTotal); + if (pSupporter->pState->code != TSDB_CODE_SUCCESS) { - tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, numOfRows, - pSupporter->pState->code); - - quitAllSubquery(pParentSql, pSupporter); + tscTrace("%p sub:%p, numOfSub:%d, quit from further procedure due to other queries failure", pParentSql, pSql, + pSupporter->subqueryIndex); + freeJoinSubqueryObj(pParentSql); return; } + + tscTrace("%p all subqueries retrieve ts complete, do ts block intersect", pParentSql); + + SJoinSupporter* p1 = pParentSql->pSubs[0]->param; + SJoinSupporter* p2 = pParentSql->pSubs[1]->param; + + TSKEY st, et; + int64_t num = doTSBlockIntersect(pParentSql, p1, p2, &st, &et); + if (num <= 0) { // no result during ts intersect + tscTrace("%p free all sub SqlObj and quit", pParentSql); + freeJoinSubqueryObj(pParentSql); + } else { + updateQueryTimeRange(pParentQueryInfo, st, et); + tscLaunchSecondPhaseSubqueries(pParentSql); + } + } +} - if (numOfRows > 0) { // write the data into disk - fwrite(pSql->res.data, pSql->res.numOfRows, 1, pSupporter->f); - fclose(pSupporter->f); - - STSBuf* pBuf = tsBufCreateFromFile(pSupporter->path, true); - if (pBuf == NULL) { - tscError("%p invalid ts comp file from vnode, abort sub query, file size:%d", pSql, numOfRows); +int32_t tagsOrderCompar(const void* p1, const void* p2) { + STidTags* t1 = (STidTags*) p1; + STidTags* t2 = (STidTags*) p2; + + if (t1->vgId != t2->vgId) { + return (t1->vgId > t2->vgId)? 1:-1; + } else { + if (t1->tid != t2->tid) { + return (t1->tid > t2->tid)? 1:-1; + } else { + return 0; + } + } +} - pSupporter->pState->code = TSDB_CODE_APP_ERROR; // todo set the informative code - quitAllSubquery(pParentSql, pSupporter); - return; +static void doBuildVgroupTableInfo(SArray* res, STableMetaInfo* pTableMetaInfo) { + SArray* pGroup = taosArrayInit(4, sizeof(SVgroupTableInfo)); + + SArray* vgTableIdItem = taosArrayInit(4, sizeof(STableIdInfo)); + int32_t size = taosArrayGetSize(res); + + STidTags* prev = taosArrayGet(res, 0); + int32_t prevVgId = prev->vgId; + + STableIdInfo item = {.uid = prev->uid, .tid = prev->tid, .key = INT64_MIN}; + taosArrayPush(vgTableIdItem, &item); + + for(int32_t k = 1; k < size; ++k) { + STidTags* t1 = taosArrayGet(res, k); + if (prevVgId != t1->vgId) { + + SVgroupTableInfo info = {0}; + + SVgroupsInfo* pvg = pTableMetaInfo->vgroupList; + for(int32_t m = 0; m < pvg->numOfVgroups; ++m) { + if (prevVgId == pvg->vgroups[m].vgId) { + info.vgInfo = pvg->vgroups[m]; + break; + } } - - if (pSupporter->pTSBuf == NULL) { - tscTrace("%p create tmp file for ts block:%s", pSql, pBuf->path); - pSupporter->pTSBuf = pBuf; - } else { - assert(pQueryInfo->numOfTables == 1); // for subquery, only one metermetaInfo - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - tsBufMerge(pSupporter->pTSBuf, pBuf, pTableMetaInfo->vgroupIndex); - tsBufDestory(pBuf); + + assert(info.vgInfo.numOfIps != 0); + info.itemList = vgTableIdItem; + taosArrayPush(pGroup, &info); + + vgTableIdItem = taosArrayInit(4, sizeof(STableIdInfo)); + STableIdInfo item1 = {.uid = t1->uid, .tid = t1->tid, .key = INT64_MIN}; + taosArrayPush(vgTableIdItem, &item1); + prevVgId = t1->vgId; + } else { + taosArrayPush(vgTableIdItem, &item); + } + } + + if (taosArrayGetSize(vgTableIdItem) > 0) { + SVgroupTableInfo info = {0}; + SVgroupsInfo* pvg = pTableMetaInfo->vgroupList; + + for(int32_t m = 0; m < pvg->numOfVgroups; ++m) { + if (prevVgId == pvg->vgroups[m].vgId) { + info.vgInfo = pvg->vgroups[m]; + break; } + } + + assert(info.vgInfo.numOfIps != 0); + info.itemList = vgTableIdItem; + taosArrayPush(pGroup, &info); + } + + pTableMetaInfo->pVgroupTables = pGroup; +} - // open new file to save the result - getTmpfilePath("ts-join", pSupporter->path); - pSupporter->f = fopen(pSupporter->path, "w"); - pSql->res.row = pSql->res.numOfRows; +static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* pParent) { + SSqlCmd* pCmd = &pSql->cmd; + tscClearSubqueryInfo(pCmd); + tscFreeSqlResult(pSql); + + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + tscInitQueryInfo(pQueryInfo); - taos_fetch_rows_a(tres, joinRetrieveCallback, param); - } else if (numOfRows == 0) { // no data from this vnode anymore - SQueryInfo* pParentQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, pParentSql->cmd.clauseIndex); + TSDB_QUERY_CLEAR_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY); + TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY); + + pCmd->command = TSDB_SQL_SELECT; + pSql->fp = tscJoinQueryCallback; + + SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = 1}; + + SColumnIndex index = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; + tscAddSpecialColumnForSelect(pQueryInfo, 0, TSDB_FUNC_TS_COMP, &index, &colSchema, TSDB_COL_NORMAL); + + // set the tags value for ts_comp function + SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, 0); + int16_t tagColIndex = tscGetJoinTagColIndexByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->uid); + + pExpr->param->i64Key = tagColIndex; + pExpr->numOfParams = 1; + + // add the filter tag column + if (pSupporter->colList != NULL) { + size_t s = taosArrayGetSize(pSupporter->colList); + + for (int32_t i = 0; i < s; ++i) { + SColumn *pCol = taosArrayGetP(pSupporter->colList, i); - //todo refactor - if (tscNonOrderedProjectionQueryOnSTable(pParentQueryInfo, 0)) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - assert(pQueryInfo->numOfTables == 1); - - // for projection query, need to try next vnode -// int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes; - int32_t totalVnode = 0; - if ((++pTableMetaInfo->vgroupIndex) < totalVnode) { - tscTrace("%p current vnode:%d exhausted, try next:%d. total vnode:%d. current numOfRes:%d", pSql, - pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVnode, pRes->numOfTotal); + if (pCol->numOfFilters > 0) { // copy to the pNew->cmd.colList if it is filtered. + SColumn *p = tscColumnClone(pCol); + taosArrayPush(pQueryInfo->colList, &p); + } + } + } - pSql->cmd.command = TSDB_SQL_SELECT; - pSql->fp = tscJoinQueryCallback; - tscProcessSql(pSql); + size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); + + tscTrace( + "%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, transfer to ts_comp query to retrieve timestamps, " + "exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s", + pParent, pSql, 0, pTableMetaInfo->vgroupIndex, pQueryInfo->type, tscSqlExprNumOfExprs(pQueryInfo), + numOfCols, pQueryInfo->fieldsInfo.numOfOutput, pTableMetaInfo->name); + + tscProcessSql(pSql); +} - return; - } - } +static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { + SJoinSupporter* pSupporter = (SJoinSupporter*)param; + + SSqlObj* pParentSql = pSupporter->pObj; + SSqlObj* pSql = (SSqlObj*)tres; + SSqlCmd* pCmd = &pSql->cmd; + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + +// if (pSupporter->pState->code != TSDB_CODE_SUCCESS) { +// tscError("%p abort query due to other subquery failure. code:%d, global code:%s", pSql, numOfRows, +// tstrerror(pSupporter->pState->code)); +// +// quitAllSubquery(pParentSql, pSupporter); +// return; +// } +// +// if (numOfRows < 0) { +// tscError("%p sub query failed, code:%s, index:%d", pSql, tstrerror(numOfRows), pSupporter->subqueryIndex); +// pSupporter->pState->code = numOfRows; +// quitAllSubquery(pParentSql, pSupporter); +// return; +// } + + // response of tag retrieve + if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) { + if (numOfRows == 0 || pSql->res.completed) { + + if (numOfRows > 0) { + size_t length = pSupporter->totalLen + pSql->res.rspLen; + char* tmp = realloc(pSupporter->pIdTagList, length); + assert(tmp != NULL); + pSupporter->pIdTagList = tmp; + + memcpy(pSupporter->pIdTagList, pSql->res.data, pSql->res.rspLen); + pSupporter->totalLen += pSql->res.rspLen; + pSupporter->num += pSql->res.numOfRows; + } + int32_t numOfTotal = pSupporter->pState->numOfTotal; int32_t finished = atomic_add_fetch_32(&pSupporter->pState->numOfCompleted, 1); - if (finished >= numOfTotal) { - assert(finished == numOfTotal); - - if (pSupporter->pState->code != TSDB_CODE_SUCCESS) { - tscTrace("%p sub:%p, numOfSub:%d, quit from further procedure due to other queries failure", pParentSql, tres, - pSupporter->subqueryIndex); - doQuitSubquery(pParentSql); - return; - } - - tscTrace("%p all subqueries retrieve ts complete, do ts block intersect", pParentSql); - - SJoinSubquerySupporter* p1 = pParentSql->pSubs[0]->param; - SJoinSubquerySupporter* p2 = pParentSql->pSubs[1]->param; - - TSKEY st, et; - - int64_t num = doTSBlockIntersect(pParentSql, p1, p2, &st, &et); - if (num <= 0) { // no result during ts intersect - tscTrace("%p free all sub SqlObj and quit", pParentSql); - doQuitSubquery(pParentSql); + if (finished < numOfTotal) { + return; + } + + // all subqueries are returned, start to compare the tags + assert(finished == numOfTotal); + tscTrace("%p all subqueries retrieve tags complete, do tags match", pParentSql); + + SJoinSupporter* p1 = pParentSql->pSubs[0]->param; + SJoinSupporter* p2 = pParentSql->pSubs[1]->param; + + qsort(p1->pIdTagList, p1->num, p1->tagSize, tagsOrderCompar); + qsort(p2->pIdTagList, p2->num, p2->tagSize, tagsOrderCompar); + + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);// todo: tags mismatch, tags not completed + + SColumn *pCol = taosArrayGetP(pTableMetaInfo->tagColList, 0); + SSchema *pColSchema = &pSchema[pCol->colIndex.columnIndex]; + + SArray* s1 = taosArrayInit(p1->num, p1->tagSize); + SArray* s2 = taosArrayInit(p2->num, p2->tagSize); + + int32_t i = 0, j = 0; + while(i < p1->num && j < p2->num) { + STidTags* pp1 = (STidTags*) p1->pIdTagList + i * p1->tagSize; + STidTags* pp2 = (STidTags*) p2->pIdTagList + j * p2->tagSize; + + int32_t ret = doCompare(pp1->tag, pp2->tag, pColSchema->type, pColSchema->bytes); + if (ret == 0) { + taosArrayPush(s1, pp1); + taosArrayPush(s2, pp2); + j++; + i++; + } else if (ret > 0) { + j++; } else { - updateQueryTimeRange(pParentQueryInfo, st, et); - tscLaunchSecondPhaseSubqueries(pParentSql); + i++; } } - } else { // failure of sub query - tscError("%p sub query failed, code:%d, index:%d", pSql, numOfRows, pSupporter->subqueryIndex); + + if (taosArrayGetSize(s1) == 0 || taosArrayGetSize(s2) == 0) {// no results,return. + tscTrace("%p free all sub SqlObj and quit", pParentSql); + freeJoinSubqueryObj(pParentSql); + return; + } else { + SSqlCmd* pSubCmd1 = &pParentSql->pSubs[0]->cmd; + SSqlCmd* pSubCmd2 = &pParentSql->pSubs[1]->cmd; + + SQueryInfo* pQueryInfo1 = tscGetQueryInfoDetail(pSubCmd1, 0); + STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo1, 0); + doBuildVgroupTableInfo(s1, pTableMetaInfo1); + + SQueryInfo* pQueryInfo2 = tscGetQueryInfoDetail(pSubCmd2, 0); + STableMetaInfo* pTableMetaInfo2 = tscGetMetaInfo(pQueryInfo2, 0); + doBuildVgroupTableInfo(s2, pTableMetaInfo2); + + pSupporter->pState->numOfCompleted = 0; + pSupporter->pState->code = 0; + pSupporter->pState->numOfTotal = 2; + + for(int32_t m = 0; m < pParentSql->numOfSubs; ++m) { + SSqlObj* psub = pParentSql->pSubs[m]; + issueTSCompQuery(psub, psub->param, pParentSql); + } + } + + } else { + size_t length = pSupporter->totalLen + pSql->res.rspLen; + char* tmp = realloc(pSupporter->pIdTagList, length); + assert(tmp != NULL); + + pSupporter->pIdTagList = tmp; + + memcpy(pSupporter->pIdTagList, pSql->res.data, pSql->res.rspLen); + pSupporter->totalLen += pSql->res.rspLen; + pSupporter->num += pSql->res.numOfRows; + + // continue retrieve data from vnode + taos_fetch_rows_a(tres, joinRetrieveCallback, param); + } + + return; + } + + if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)) { + if (numOfRows < 0) { + tscError("%p sub query failed, code:%s, index:%d", pSql, tstrerror(numOfRows), pSupporter->subqueryIndex); pSupporter->pState->code = numOfRows; + quitAllSubquery(pParentSql, pSupporter); + return; + } + + if (numOfRows == 0) { + tSIntersectionAndLaunchSecQuery(pSupporter, pSql); + return; + } + + // write the compressed timestamp to disk file + fwrite(pSql->res.data, pSql->res.numOfRows, 1, pSupporter->f); + fclose(pSupporter->f); + pSupporter->f = NULL; + + STSBuf* pBuf = tsBufCreateFromFile(pSupporter->path, true); + if (pBuf == NULL) { + tscError("%p invalid ts comp file from vnode, abort subquery, file size:%d", pSql, numOfRows); + pSupporter->pState->code = TSDB_CODE_APP_ERROR; // todo set the informative code quitAllSubquery(pParentSql, pSupporter); return; } + if (pSupporter->pTSBuf == NULL) { + tscTrace("%p create tmp file for ts block:%s, size:%d bytes", pSql, pBuf->path, numOfRows); + pSupporter->pTSBuf = pBuf; + } else { + assert(pQueryInfo->numOfTables == 1); // for subquery, only one + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + tsBufMerge(pSupporter->pTSBuf, pBuf, pTableMetaInfo->vgroupIndex); + tsBufDestory(pBuf); + } + + if (pSql->res.completed) { + tSIntersectionAndLaunchSecQuery(pSupporter, pSql); + } else { // open a new file to save the incoming result + getTmpfilePath("ts-join", pSupporter->path); + pSupporter->f = fopen(pSupporter->path, "w"); + pSql->res.row = pSql->res.numOfRows; + + taos_fetch_rows_a(tres, joinRetrieveCallback, param); + } } else { // secondary stage retrieve, driven by taos_fetch_row or other functions if (numOfRows < 0) { pSupporter->pState->code = numOfRows; - tscError("%p retrieve failed, code:%d, index:%d", pSql, numOfRows, pSupporter->subqueryIndex); + tscError("%p retrieve failed, code:%s, index:%d", pSql, tstrerror(numOfRows), pSupporter->subqueryIndex); } if (numOfRows >= 0) { @@ -542,20 +778,20 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { } if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && numOfRows == 0) { -// STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); assert(pQueryInfo->numOfTables == 1); - - // for projection query, need to try next vnode if current vnode is exhausted -// if ((++pTableMetaInfo->vgroupIndex) < pTableMetaInfo->pMetricMeta->numOfVnodes) { -// pSupporter->pState->numOfCompleted = 0; -// pSupporter->pState->numOfTotal = 1; -// -// pSql->cmd.command = TSDB_SQL_SELECT; -// pSql->fp = tscJoinQueryCallback; -// tscProcessSql(pSql); -// -// return; -// } + + // for projection query, need to try next vnode if current vnode is exhausted + if ((++pTableMetaInfo->vgroupIndex) < pTableMetaInfo->vgroupList->numOfVgroups) { + pSupporter->pState->numOfCompleted = 0; + pSupporter->pState->numOfTotal = 1; + + pSql->cmd.command = TSDB_SQL_SELECT; + pSql->fp = tscJoinQueryCallback; + tscProcessSql(pSql); + + return; + } } int32_t numOfTotal = pSupporter->pState->numOfTotal; @@ -567,27 +803,39 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { pParentSql->res.code); if (pSupporter->pState->code != TSDB_CODE_SUCCESS) { - pParentSql->res.code = abs(pSupporter->pState->code); - freeSubqueryObj(pParentSql); + pParentSql->res.code = pSupporter->pState->code; + freeJoinSubqueryObj(pParentSql); + pParentSql->res.completed = true; } - -// tsem_post(&pParentSql->rspSem); + + // update the records for each subquery in parent sql object. + for(int32_t i = 0; i < pParentSql->numOfSubs; ++i) { + if (pParentSql->pSubs[i] == NULL) { + continue; + } + + SSqlRes* pRes1 = &pParentSql->pSubs[i]->res; + pRes1->numOfTotalInCurrentClause += pRes1->numOfRows; + } + + // data has retrieved to client, build the join results + tscBuildResFromSubqueries(pParentSql); } else { tscTrace("%p sub:%p completed, completed:%d, total:%d", pParentSql, tres, finished, numOfTotal); } } } -static SJoinSubquerySupporter* tscUpdateSubqueryStatus(SSqlObj* pSql, int32_t numOfFetch) { +static SJoinSupporter* tscUpdateSubqueryStatus(SSqlObj* pSql, int32_t numOfFetch) { int32_t notInvolved = 0; - SJoinSubquerySupporter* pSupporter = NULL; + SJoinSupporter* pSupporter = NULL; SSubqueryState* pState = NULL; for(int32_t i = 0; i < pSql->numOfSubs; ++i) { if (pSql->pSubs[i] == NULL) { notInvolved++; } else { - pSupporter = (SJoinSubquerySupporter*)pSql->pSubs[i]->param; + pSupporter = (SJoinSupporter*)pSql->pSubs[i]->param; pState = pSupporter->pState; } } @@ -599,39 +847,62 @@ static SJoinSubquerySupporter* tscUpdateSubqueryStatus(SSqlObj* pSql, int32_t nu } void tscFetchDatablockFromSubquery(SSqlObj* pSql) { - int32_t numOfFetch = 0; assert(pSql->numOfSubs >= 1); + int32_t numOfFetch = 0; + bool hasData = true; for (int32_t i = 0; i < pSql->numOfSubs; ++i) { - if (pSql->pSubs[i] == NULL) { // this subquery does not need to involve in secondary query + // if the subquery is NULL, it does not involved in the final result generation + SSqlObj* pSub = pSql->pSubs[i]; + if (pSub == NULL) { continue; } - SSqlRes *pRes = &pSql->pSubs[i]->res; - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->pSubs[i]->cmd, 0); - + SSqlRes *pRes = &pSub->res; + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSub->cmd, 0); // STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { -// if (pRes->row >= pRes->numOfRows && pTableMetaInfo->vgroupIndex < pTableMetaInfo->pMetricMeta->numOfVnodes && -// (!tscHasReachLimitation(pQueryInfo, pRes))) { +// if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { +// if (pRes->row >= pRes->numOfRows && pTableMetaInfo->vgroupIndex < pTableMetaInfo->vgroupList->numOfVgroups && +// (!tscHasReachLimitation(pQueryInfo, pRes)) && !pRes->completed) { // numOfFetch++; // } - } else { - if (pRes->row >= pRes->numOfRows && (!tscHasReachLimitation(pQueryInfo, pRes))) { - numOfFetch++; +// } else { + if (!tscHasReachLimitation(pQueryInfo, pRes)) { + if (pRes->row >= pRes->numOfRows) { + hasData = false; + + if (!pRes->completed) { + numOfFetch++; + } + } + } else { // has reach the limitation, no data anymore + hasData = false; } + } - } +// } - if (numOfFetch <= 0) { + // has data remains in client side, and continue to return data to app + if (hasData) { + tscBuildResFromSubqueries(pSql); + return; + } else if (numOfFetch <= 0) { + pSql->res.completed = true; + freeJoinSubqueryObj(pSql); + + if (pSql->res.code == TSDB_CODE_SUCCESS) { + (*pSql->fp)(pSql->param, pSql, 0); + } else { + tscQueueAsyncRes(pSql); + } + return; } // TODO multi-vnode retrieve for projection query with limitation has bugs, since the global limiation is not handled tscTrace("%p retrieve data from %d subqueries", pSql, numOfFetch); - - SJoinSubquerySupporter* pSupporter = tscUpdateSubqueryStatus(pSql, numOfFetch); + SJoinSupporter* pSupporter = tscUpdateSubqueryStatus(pSql, numOfFetch); for (int32_t i = 0; i < pSql->numOfSubs; ++i) { SSqlObj* pSql1 = pSql->pSubs[i]; @@ -642,7 +913,7 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) { SSqlRes* pRes1 = &pSql1->res; SSqlCmd* pCmd1 = &pSql1->cmd; - pSupporter = (SJoinSubquerySupporter*)pSql1->param; + pSupporter = (SJoinSupporter*)pSql1->param; // wait for all subqueries completed SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd1, 0); @@ -664,19 +935,6 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) { tscProcessSql(pSql1); } } - - // wait for all subquery completed -// tsem_wait(&pSql->rspSem); - - // update the records for each subquery - for(int32_t i = 0; i < pSql->numOfSubs; ++i) { - if (pSql->pSubs[i] == NULL) { - continue; - } - - SSqlRes* pRes1 = &pSql->pSubs[i]->res; - pRes1->numOfTotalInCurrentClause += pRes1->numOfRows; - } } // all subqueries return, set the result output index @@ -710,7 +968,7 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) { SSqlCmd* pSubCmd = &pSql->pSubs[tableIndexOfSub]->cmd; SQueryInfo* pSubQueryInfo = tscGetQueryInfoDetail(pSubCmd, 0); - size_t numOfExprs = taosArrayGetSize(pSubQueryInfo->exprsInfo); + size_t numOfExprs = taosArrayGetSize(pSubQueryInfo->exprList); for (int32_t k = 0; k < numOfExprs; ++k) { SSqlExpr* pSubExpr = tscSqlExprGet(pSubQueryInfo, k); if (pExpr->functionId == pSubExpr->functionId && pExpr->colInfo.colId == pSubExpr->colInfo.colId) { @@ -723,35 +981,17 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) { void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { SSqlObj* pSql = (SSqlObj*)tres; - // STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - - // int32_t idx = pSql->cmd.vnodeIdx; - - SJoinSubquerySupporter* pSupporter = (SJoinSubquerySupporter*)param; - - // if (atomic_add_fetch_32(pSupporter->numOfComplete, 1) >= - // pSupporter->numOfTotal) { - // SSqlObj *pParentObj = pSupporter->pObj; - // - // if ((pSql->cmd.type & TSDB_QUERY_TYPE_JOIN_SEC_STAGE) != 1) { - // int32_t num = 0; - // tscFetchDatablockFromSubquery(pParentObj); - // TSKEY* ts = tscGetQualifiedTSList(pParentObj, &num); - // - // if (num <= 0) { - // // no qualified result - // } - // - // tscLaunchSecondPhaseSubqueries(pSql, ts, num); - // } else { - - // } - // } else { + + SJoinSupporter* pSupporter = (SJoinSupporter*)param; + + // There is only one subquery and table for each subquery. SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - if ((pQueryInfo->type & TSDB_QUERY_TYPE_JOIN_SEC_STAGE) != TSDB_QUERY_TYPE_JOIN_SEC_STAGE) { + assert(pQueryInfo->numOfTables == 1 && pSql->cmd.numOfClause == 1); + + if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)) { if (code != TSDB_CODE_SUCCESS) { // direct call joinRetrieveCallback and set the error code joinRetrieveCallback(param, pSql, code); - } else { // first stage query, continue to retrieve data + } else { // first stage query, continue to retrieve compressed time stamp data pSql->fp = joinRetrieveCallback; pSql->cmd.command = TSDB_SQL_FETCH; tscProcessSql(pSql); @@ -769,10 +1009,10 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { } if (code != TSDB_CODE_SUCCESS) { - tscError("%p sub query failed, code:%d, set global code:%d, index:%d", pSql, code, code, + tscError("%p sub query failed, code:%s, set global code:%s, index:%d", pSql, tstrerror(code), tstrerror(code), pSupporter->subqueryIndex); - pSupporter->pState->code = code; // todo set the informative code - + + pSupporter->pState->code = code; quitAllSubquery(pParentSql, pSupporter); } else { int32_t numOfTotal = pSupporter->pState->numOfTotal; @@ -789,24 +1029,17 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { * data instead of returning to its invoker */ if (pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { -// assert(pTableMetaInfo->vgroupIndex < pTableMetaInfo->pMetricMeta->numOfVnodes); pSupporter->pState->numOfCompleted = 0; // reset the record value pSql->fp = joinRetrieveCallback; // continue retrieve data pSql->cmd.command = TSDB_SQL_FETCH; tscProcessSql(pSql); } else { // first retrieve from vnode during the secondary stage sub-query - if (pParentSql->fp == NULL) { -// tsem_post(&pParentSql->rspSem); + // set the command flag must be after the semaphore been correctly set. + if (pParentSql->res.code == TSDB_CODE_SUCCESS) { + (*pParentSql->fp)(pParentSql->param, pParentSql, 0); } else { - // set the command flag must be after the semaphore been correctly set. - // pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC; - // if (pPObj->res.code == TSDB_CODE_SUCCESS) { - // (*pPObj->fp)(pPObj->param, pPObj, 0); - // } else { - // tscQueueAsyncRes(pPObj); - // } - assert(0); + tscQueueAsyncRes(pParentSql); } } } @@ -820,7 +1053,7 @@ static void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code); static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj); // todo merge with callback -int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSubquerySupporter *pSupporter) { +int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter *pSupporter) { SSqlCmd * pCmd = &pSql->cmd; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); @@ -856,12 +1089,19 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSubquerySu pCol->colIndex.tableIndex = 0; } - tscColumnListCopy(pSupporter->colList, pNewQueryInfo->colList, 0); + pSupporter->colList = pNewQueryInfo->colList; + pNewQueryInfo->colList = NULL; + + pSupporter->exprList = pNewQueryInfo->exprList; + pNewQueryInfo->exprList = NULL; + + pSupporter->fieldsInfo = pNewQueryInfo->fieldsInfo; - pSupporter->exprsInfo = tscSqlExprCopy(pNewQueryInfo->exprsInfo, pSupporter->uid, false); - tscFieldInfoCopy(&pSupporter->fieldsInfo, &pNewQueryInfo->fieldsInfo); + // this data needs to be transfer to support struct + memset(&pNewQueryInfo->fieldsInfo, 0, sizeof(SFieldInfo)); - tscTagCondCopy(&pSupporter->tagCond, &pNewQueryInfo->tagCond); + pSupporter->tagCond = pNewQueryInfo->tagCond; + memset(&pNewQueryInfo->tagCond, 0, sizeof(STagCond)); pNew->cmd.numOfCols = 0; pNewQueryInfo->intervalTime = 0; @@ -871,72 +1111,98 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSubquerySu pSupporter->groupbyExpr = pNewQueryInfo->groupbyExpr; memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SSqlGroupbyExpr)); - // this data needs to be transfer to support struct - pNewQueryInfo->fieldsInfo.numOfOutput = 0; - - // set the ts,tags that involved in join, as the output column of intermediate result - tscClearSubqueryInfo(&pNew->cmd); - - SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = 1}; - SColumnIndex index = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - - tscAddSpecialColumnForSelect(pNewQueryInfo, 0, TSDB_FUNC_TS_COMP, &index, &colSchema, TSDB_COL_NORMAL); - - // set the tags value for ts_comp function - SSqlExpr *pExpr = tscSqlExprGet(pNewQueryInfo, 0); - + tscInitQueryInfo(pNewQueryInfo); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0); - int16_t tagColIndex = tscGetJoinTagColIndexByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->uid); - pExpr->param->i64Key = tagColIndex; - pExpr->numOfParams = 1; - - // add the filter tag column - size_t s = taosArrayGetSize(pSupporter->colList); - - for (int32_t i = 0; i < s; ++i) { - SColumn *pCol = taosArrayGetP(pSupporter->colList, i); - - if (pCol->numOfFilters > 0) { // copy to the pNew->cmd.colList if it is filtered. - SColumn* p = tscColumnClone(pCol); - taosArrayPush(pNewQueryInfo->colList, &p); + if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { // return the tableId & tag + SSchema s = {0}; + SColumnIndex index = {0}; + + size_t numOfTags = taosArrayGetSize(pTableMetaInfo->tagColList); + for(int32_t i = 0; i < numOfTags; ++i) { + SColumn* c = taosArrayGetP(pTableMetaInfo->tagColList, i); + index = (SColumnIndex) {.tableIndex = 0, .columnIndex = c->colIndex.columnIndex}; + + SSchema* pTagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); + s = pTagSchema[c->colIndex.columnIndex]; + + int16_t bytes = 0; + int16_t type = 0; + int16_t inter = 0; + + getResultDataInfo(s.type, s.bytes, TSDB_FUNC_TID_TAG, 0, &type, &bytes, &inter, 0, 0); + + s.type = type; + s.bytes = bytes; + pSupporter->tagSize = s.bytes; } - } - size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList); + // set get tags query type + TSDB_QUERY_SET_TYPE(pNewQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY); + tscAddSpecialColumnForSelect(pNewQueryInfo, 0, TSDB_FUNC_TID_TAG, &index, &s, TSDB_COL_TAG); + size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList); - tscTrace("%p subquery:%p tableIndex:%d, vnodeIdx:%d, type:%d, transfer to ts_comp query to retrieve timestamps, " - "exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s", - pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, - tscSqlExprNumOfExprs(pNewQueryInfo), numOfCols, - pNewQueryInfo->fieldsInfo.numOfOutput, pNewQueryInfo->pTableMetaInfo[0]->name); - tscPrintSelectClause(pNew, 0); + tscTrace( + "%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, transfer to tid_tag query to retrieve (tableId, tags), " + "exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s", + pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscSqlExprNumOfExprs(pNewQueryInfo), + numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, pNewQueryInfo->pTableMetaInfo[0]->name); + + } else { + SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = 1}; + SColumnIndex index = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; + tscAddSpecialColumnForSelect(pNewQueryInfo, 0, TSDB_FUNC_TS_COMP, &index, &colSchema, TSDB_COL_NORMAL); + + // set the tags value for ts_comp function + SSqlExpr *pExpr = tscSqlExprGet(pNewQueryInfo, 0); + + int16_t tagColIndex = tscGetJoinTagColIndexByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->uid); + pExpr->param->i64Key = tagColIndex; + pExpr->numOfParams = 1; + + // add the filter tag column + if (pSupporter->colList != NULL) { + size_t s = taosArrayGetSize(pSupporter->colList); + + for (int32_t i = 0; i < s; ++i) { + SColumn *pCol = taosArrayGetP(pSupporter->colList, i); + + if (pCol->numOfFilters > 0) { // copy to the pNew->cmd.colList if it is filtered. + SColumn *p = tscColumnClone(pCol); + taosArrayPush(pNewQueryInfo->colList, &p); + } + } + } + + size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList); + + tscTrace( + "%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, transfer to ts_comp query to retrieve timestamps, " + "exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s", + pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscSqlExprNumOfExprs(pNewQueryInfo), + numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, pNewQueryInfo->pTableMetaInfo[0]->name); + } } else { + assert(0); SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); pNewQueryInfo->type |= TSDB_QUERY_TYPE_SUBQUERY; } -#ifdef _DEBUG_VIEW - tscPrintSelectClause(pNew, 0); -#endif - return tscProcessSql(pNew); } -// todo support async join query int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - assert((pQueryInfo->type & TSDB_QUERY_TYPE_SUBQUERY) == 0); SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); - pState->numOfTotal = pQueryInfo->numOfTables; + tscTrace("%p start launch subquery, total:%d", pSql, pQueryInfo->numOfTables); for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - SJoinSubquerySupporter *pSupporter = tscCreateJoinSupporter(pSql, pState, i); + SJoinSupporter *pSupporter = tscCreateJoinSupporter(pSql, pState, i); if (pSupporter == NULL) { // failed to create support struct, abort current query tscError("%p tableIndex:%d, failed to allocate join support object, abort further query", pSql, i); @@ -955,13 +1221,7 @@ int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) { } } -// tsem_wait(&pSql->rspSem); - - if (pSql->numOfSubs <= 0) { - pSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; - } else { - pSql->cmd.command = TSDB_SQL_METRIC_JOIN_RETRIEVE; - } + pSql->cmd.command = (pSql->numOfSubs <= 0)? TSDB_SQL_RETRIEVE_EMPTY_RESULT:TSDB_SQL_METRIC_JOIN_RETRIEVE; return TSDB_CODE_SUCCESS; } @@ -994,7 +1254,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { // pRes->code check only serves in launching metric sub-queries if (pRes->code == TSDB_CODE_QUERY_CANCELLED) { - pCmd->command = TSDB_SQL_RETRIEVE_METRIC; // enable the abort of kill metric function. + pCmd->command = TSDB_SQL_RETRIEVE_LOCALMERGE; // enable the abort of kill super table function. return pRes->code; } @@ -1015,10 +1275,8 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, nBufferSize); if (ret != 0) { pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; - if (pSql->fp) { - tscQueueAsyncRes(pSql); - } - return pRes->code; + tscQueueAsyncRes(pSql); + return ret; } pSql->pSubs = calloc(pSql->numOfSubs, POINTER_BYTES); @@ -1051,7 +1309,9 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { trs->pParentSqlObj = pSql; trs->pFinalColModel = pModel; - pthread_mutexattr_t mutexattr = {0}; + pthread_mutexattr_t mutexattr; + memset(&mutexattr, 0, sizeof(pthread_mutexattr_t)); + pthread_mutexattr_settype(&mutexattr, PTHREAD_MUTEX_RECURSIVE_NP); pthread_mutex_init(&trs->queryMutex, &mutexattr); pthread_mutexattr_destroy(&mutexattr); @@ -1127,9 +1387,7 @@ static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES tscError("sub:%p failed to flush data to disk:reason:%s", tres, lpMsgBuf); LocalFree(lpMsgBuf); #else - char buf[256] = {0}; - strerror_r(errno, buf, 256); - tscError("sub:%p failed to flush data to disk:reason:%s", tres, buf); + tscError("sub:%p failed to flush data to disk:reason:%s", tres, strerror(errno)); #endif trsupport->pState->code = -errCode; @@ -1179,8 +1437,8 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO trsupport->localBuffer->numOfElems = 0; pthread_mutex_unlock(&trsupport->queryMutex); - tscTrace("%p sub:%p retrieve failed, code:%d, orderOfSub:%d, retry:%d", trsupport->pParentSqlObj, pSql, numOfRows, - subqueryIndex, trsupport->numOfRetry); + tscTrace("%p sub:%p retrieve failed, code:%s, orderOfSub:%d, retry:%d", trsupport->pParentSqlObj, pSql, + tstrerror(numOfRows), subqueryIndex, trsupport->numOfRetry); SSqlObj *pNew = tscCreateSqlObjForSubquery(trsupport->pParentSqlObj, trsupport, pSql); if (pNew == NULL) { @@ -1196,8 +1454,8 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO return; } else { // reach the maximum retry count, abort atomic_val_compare_exchange_32(&pState->code, TSDB_CODE_SUCCESS, numOfRows); - tscError("%p sub:%p retrieve failed,code:%d,orderOfSub:%d failed.no more retry,set global code:%d", pPObj, pSql, - numOfRows, subqueryIndex, pState->code); + tscError("%p sub:%p retrieve failed,code:%s,orderOfSub:%d failed.no more retry,set global code:%d", pPObj, pSql, + numOfRows, subqueryIndex, tstrerror(pState->code)); } } @@ -1306,7 +1564,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p tscFreeSubSqlObj(trsupport, pSql); // set the command flag must be after the semaphore been correctly set. - pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC; + pPObj->cmd.command = TSDB_SQL_RETRIEVE_LOCALMERGE; if (pPObj->res.code == TSDB_CODE_SUCCESS) { (*pPObj->fp)(pPObj->param, pPObj, 0); } else { @@ -1576,3 +1834,233 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { return TSDB_CODE_SUCCESS; } + +void tscBuildResFromSubqueries(SSqlObj *pSql) { + SSqlRes *pRes = &pSql->res; + + if (pRes->code != TSDB_CODE_SUCCESS) { + tscQueueAsyncRes(pSql); + return; + } + + while (1) { + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + + if (pRes->tsrow == NULL) { + pRes->tsrow = calloc(numOfExprs, POINTER_BYTES); + } + + bool success = false; + + int32_t numOfTableHasRes = 0; + for (int32_t i = 0; i < pSql->numOfSubs; ++i) { + if (pSql->pSubs[i] != NULL) { + numOfTableHasRes++; + } + } + + if (numOfTableHasRes >= 2) { // do merge result + success = (doSetResultRowData(pSql->pSubs[0], false) != NULL) && (doSetResultRowData(pSql->pSubs[1], false) != NULL); + } else { // only one subquery + SSqlObj *pSub = pSql->pSubs[0]; + if (pSub == NULL) { + pSub = pSql->pSubs[1]; + } + + success = (doSetResultRowData(pSub, false) != NULL); + } + + if (success) { // current row of final output has been built, return to app + for (int32_t i = 0; i < numOfExprs; ++i) { + SColumnIndex* pIndex = &pRes->pColumnIndex[i]; + SSqlRes *pRes1 = &pSql->pSubs[pIndex->tableIndex]->res; + pRes->tsrow[i] = pRes1->tsrow[pIndex->columnIndex]; + } + + pRes->numOfTotalInCurrentClause++; + break; + } else { // continue retrieve data from vnode + if (!tscHashRemainDataInSubqueryResultSet(pSql)) { + tscTrace("%p at least one subquery exhausted, free all other %d subqueries", pSql, pSql->numOfSubs - 1); + SSubqueryState *pState = NULL; + + // free all sub sqlobj + for (int32_t i = 0; i < pSql->numOfSubs; ++i) { + SSqlObj *pChildObj = pSql->pSubs[i]; + if (pChildObj == NULL) { + continue; + } + + SJoinSupporter *pSupporter = (SJoinSupporter *)pChildObj->param; + pState = pSupporter->pState; + + tscDestroyJoinSupporter(pChildObj->param); + taos_free_result(pChildObj); + } + + free(pState); + + pRes->completed = true; // set query completed + sem_post(&pSql->rspSem); + return; + } + + tscFetchDatablockFromSubquery(pSql); + if (pRes->code != TSDB_CODE_SUCCESS) { + return; + } + } + } + + if (pSql->res.code == TSDB_CODE_SUCCESS) { + (*pSql->fp)(pSql->param, pSql, 0); + } else { + tscQueueAsyncRes(pSql); + } +} + +static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pField) { + SSqlRes *pRes = &pSql->res; + + if (isNull(pRes->tsrow[columnIndex], pField->type)) { + pRes->tsrow[columnIndex] = NULL; + } else if (pField->type == TSDB_DATA_TYPE_NCHAR) { + // convert unicode to native code in a temporary buffer extra one byte for terminated symbol + if (pRes->buffer[columnIndex] == NULL) { + pRes->buffer[columnIndex] = malloc(pField->bytes + TSDB_NCHAR_SIZE); + } + + /* string terminated char for binary data*/ + memset(pRes->buffer[columnIndex], 0, pField->bytes + TSDB_NCHAR_SIZE); + + if (taosUcs4ToMbs(pRes->tsrow[columnIndex], pField->bytes - VARSTR_HEADER_SIZE, pRes->buffer[columnIndex])) { + pRes->tsrow[columnIndex] = pRes->buffer[columnIndex]; + } else { + tscError("%p charset:%s to %s. val:%ls convert failed.", pSql, DEFAULT_UNICODE_ENCODEC, tsCharset, pRes->tsrow); + pRes->tsrow[columnIndex] = NULL; + } + } +} + +void **doSetResultRowData(SSqlObj *pSql, bool finalResult) { + SSqlCmd *pCmd = &pSql->cmd; + SSqlRes *pRes = &pSql->res; + + assert(pRes->row >= 0 && pRes->row <= pRes->numOfRows); + + if(pCmd->command == TSDB_SQL_METRIC_JOIN_RETRIEVE) { + if (pRes->completed) { + tfree(pRes->tsrow); + } + + return pRes->tsrow; + } + + if (pRes->row >= pRes->numOfRows) { // all the results has returned to invoker + tfree(pRes->tsrow); + return pRes->tsrow; + } + + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + + for (int i = 0; i < tscNumOfFields(pQueryInfo); ++i) { + SFieldSupInfo* pSup = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, i); + if (pSup->pSqlExpr != NULL) { + pRes->tsrow[i] = tscGetResultColumnChr(pRes, pQueryInfo, i, pSup->pSqlExpr->resBytes); + } + + // primary key column cannot be null in interval query, no need to check + if (i == 0 && pQueryInfo->intervalTime > 0) { + continue; + } + + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); + transferNcharData(pSql, i, pField); + + // calculate the result from several other columns + if (pSup->pArithExprInfo != NULL) { +// SArithmeticSupport *sas = (SArithmeticSupport *)calloc(1, sizeof(SArithmeticSupport)); +// sas->offset = 0; +// sas-> = pQueryInfo->fieldsInfo.pExpr[i]; +// +// sas->numOfCols = sas->pExpr->binExprInfo.numOfCols; +// +// if (pRes->buffer[i] == NULL) { +// pRes->buffer[i] = malloc(tscFieldInfoGetField(pQueryInfo, i)->bytes); +// } +// +// for(int32_t k = 0; k < sas->numOfCols; ++k) { +// int32_t columnIndex = sas->pExpr->binExprInfo.pReqColumns[k].colIdxInBuf; +// SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, columnIndex); +// +// sas->elemSize[k] = pExpr->resBytes; +// sas->data[k] = (pRes->data + pRes->numOfRows* pExpr->offset) + pRes->row*pExpr->resBytes; +// } +// +// tSQLBinaryExprCalcTraverse(sas->pExpr->binExprInfo.pBinExpr, 1, pRes->buffer[i], sas, TSQL_SO_ASC, getArithemicInputSrc); +// pRes->tsrow[i] = pRes->buffer[i]; +// +// free(sas); //todo optimization + } + } + + pRes->row++; // index increase one-step + return pRes->tsrow; +} + +static bool tscHashRemainDataInSubqueryResultSet(SSqlObj *pSql) { + bool hasData = true; + SSqlCmd *pCmd = &pSql->cmd; + + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { + bool allSubqueryExhausted = true; + + for (int32_t i = 0; i < pSql->numOfSubs; ++i) { + if (pSql->pSubs[i] == NULL) { + continue; + } + + SSqlRes *pRes1 = &pSql->pSubs[i]->res; + SSqlCmd *pCmd1 = &pSql->pSubs[i]->cmd; + + SQueryInfo *pQueryInfo1 = tscGetQueryInfoDetail(pCmd1, pCmd1->clauseIndex); + assert(pQueryInfo1->numOfTables == 1); + + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo1, 0); + + /* + * if the global limitation is not reached, and current result has not exhausted, or next more vnodes are + * available, goes on + */ + if (pTableMetaInfo->vgroupIndex < pTableMetaInfo->vgroupList->numOfVgroups && pRes1->row < pRes1->numOfRows && + (!tscHasReachLimitation(pQueryInfo1, pRes1))) { + allSubqueryExhausted = false; + break; + } + } + + hasData = !allSubqueryExhausted; + } else { // otherwise, in case inner join, if any subquery exhausted, query completed. + for (int32_t i = 0; i < pSql->numOfSubs; ++i) { + if (pSql->pSubs[i] == 0) { + continue; + } + + SSqlRes * pRes1 = &pSql->pSubs[i]->res; + SQueryInfo *pQueryInfo1 = tscGetQueryInfoDetail(&pSql->pSubs[i]->cmd, 0); + + if ((pRes1->row >= pRes1->numOfRows && tscHasReachLimitation(pQueryInfo1, pRes1) && + tscProjectionQueryOnTable(pQueryInfo1)) || + (pRes1->numOfRows == 0)) { + hasData = false; + break; + } + } + } + + return hasData; +} + + diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 506fa1a60560baf5da1e5f401a2ad0798651fcc3..d07185b0ba798060818e86bd0160ad279ff6bf36 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -76,7 +76,9 @@ bool tscQueryOnSTable(SSqlCmd* pCmd) { bool tscQueryTags(SQueryInfo* pQueryInfo) { for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - if (tscSqlExprGet(pQueryInfo, i)->functionId != TSDB_FUNC_TAGPRJ) { + int32_t functId = tscSqlExprGet(pQueryInfo, i)->functionId; + + if (functId != TSDB_FUNC_TAGPRJ && functId != TSDB_FUNC_TID_TAG) { return false; } } @@ -123,23 +125,23 @@ void tscGetDBInfoFromMeterId(char* tableId, char* db) { db[0] = 0; } -STableIdInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx) { - if (pSidList == NULL) { - tscError("illegal sidlist"); - return 0; - } - - if (idx < 0 || idx >= pSidList->numOfSids) { - int32_t sidRange = (pSidList->numOfSids > 0) ? (pSidList->numOfSids - 1) : 0; - - tscError("illegal sidIdx:%d, reset to 0, sidIdx range:%d-%d", idx, 0, sidRange); - idx = 0; - } - - assert(pSidList->pSidExtInfoList[idx] >= 0); - - return (STableIdInfo*)(pSidList->pSidExtInfoList[idx] + (char*)pSidList); -} +//STableIdInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx) { +// if (pSidList == NULL) { +// tscError("illegal sidlist"); +// return 0; +// } +// +// if (idx < 0 || idx >= pSidList->numOfSids) { +// int32_t sidRange = (pSidList->numOfSids > 0) ? (pSidList->numOfSids - 1) : 0; +// +// tscError("illegal sidIdx:%d, reset to 0, sidIdx range:%d-%d", idx, 0, sidRange); +// idx = 0; +// } +// +// assert(pSidList->pSidExtInfoList[idx] >= 0); +// +// return (STableIdInfo*)(pSidList->pSidExtInfoList[idx] + (char*)pSidList); +//} bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) { if (pQueryInfo == NULL) { @@ -151,15 +153,11 @@ bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) { return false; } - // for select query super table, the metricmeta can not be null in any cases. + // for select query super table, the super table vgroup list can not be null in any cases. if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { -// assert(pTableMetaInfo->pMetricMeta != NULL); + assert(pTableMetaInfo->vgroupList != NULL); } -// if (pTableMetaInfo->pMetricMeta == NULL) { -// return false; -// } - if ((pQueryInfo->type & TSDB_QUERY_TYPE_FREE_RESOURCE) == TSDB_QUERY_TYPE_FREE_RESOURCE) { return false; } @@ -191,12 +189,11 @@ bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) { return false; } - // only query on tag, not a projection query + // only query on tag, a project query if (tscQueryTags(pQueryInfo)) { - return false; + return true; } - // for project query, only the following two function is allowed for (int32_t i = 0; i < numOfExprs; ++i) { int32_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; if (functionId != TSDB_FUNC_PRJ && functionId != TSDB_FUNC_TAGPRJ && functionId != TSDB_FUNC_TAG && @@ -306,12 +303,10 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) { } void tscDestroyResPointerInfo(SSqlRes* pRes) { - if (pRes->buffer != NULL) { - // free all buffers containing the multibyte string + if (pRes->buffer != NULL) { // free all buffers containing the multibyte string for (int i = 0; i < pRes->numOfCols; i++) { tfree(pRes->buffer[i]); } - pRes->numOfCols = 0; } @@ -340,50 +335,15 @@ void tscResetSqlCmdObj(SSqlCmd* pCmd) { tscFreeQueryInfo(pCmd); } -/* - * this function must not change the pRes->code value, since it may be used later. - */ -void tscFreeResData(SSqlObj* pSql) { - SSqlRes* pRes = &pSql->res; - - pRes->row = 0; - - pRes->rspType = 0; - pRes->rspLen = 0; - pRes->row = 0; - - pRes->numOfRows = 0; - pRes->numOfTotal = 0; - pRes->numOfTotalInCurrentClause = 0; - - pRes->numOfGroups = 0; - pRes->precision = 0; - pRes->qhandle = 0; - - pRes->offset = 0; - pRes->useconds = 0; - +void tscFreeSqlResult(SSqlObj* pSql) { tscDestroyLocalReducer(pSql); + SSqlRes* pRes = &pSql->res; tscDestroyResPointerInfo(pRes); + memset(&pSql->res, 0, sizeof(SSqlRes)); } -void tscFreeSqlResult(SSqlObj* pSql) { - tfree(pSql->res.pRsp); - pSql->res.row = 0; - pSql->res.numOfRows = 0; - pSql->res.numOfTotal = 0; - - pSql->res.numOfGroups = 0; - tfree(pSql->res.pGroupRec); - - tscDestroyLocalReducer(pSql); - - tscDestroyResPointerInfo(&pSql->res); - tfree(pSql->res.pColumnIndex); -} - -void tscFreeSqlObjPartial(SSqlObj* pSql) { +void tscPartiallyFreeSqlObj(SSqlObj* pSql) { if (pSql == NULL || pSql->signature != pSql) { return; } @@ -392,7 +352,7 @@ void tscFreeSqlObjPartial(SSqlObj* pSql) { STscObj* pObj = pSql->pTscObj; int32_t cmd = pCmd->command; - if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_METRIC || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT || + if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_LOCALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT || cmd == TSDB_SQL_METRIC_JOIN_RETRIEVE) { tscRemoveFromSqlList(pSql); } @@ -417,7 +377,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { if (pSql == NULL || pSql->signature != pSql) return; tscTrace("%p start to free sql object", pSql); - tscFreeSqlObjPartial(pSql); + tscPartiallyFreeSqlObj(pSql); pSql->signature = NULL; pSql->fp = NULL; @@ -643,14 +603,16 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock) { memcpy(pDataBlock, pTableDataBlock->pData, sizeof(SSubmitBlk)); pDataBlock += sizeof(SSubmitBlk); - int32_t flen = 0; + int32_t flen = 0; // original total length of row for (int32_t i = 0; i < tinfo.numOfColumns; ++i) { flen += TYPE_BYTES[pSchema[i].type]; } char* p = pTableDataBlock->pData + sizeof(SSubmitBlk); pBlock->len = 0; - for (int32_t i = 0; i < htons(pBlock->numOfRows); ++i) { + int32_t numOfRows = htons(pBlock->numOfRows); + + for (int32_t i = 0; i < numOfRows; ++i) { SDataRow trow = (SDataRow)pDataBlock; dataRowSetLen(trow, TD_DATA_ROW_HEAD_SIZE + flen); @@ -748,27 +710,27 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi } void tscCloseTscObj(STscObj* pObj) { + assert(pObj != NULL); + pObj->signature = NULL; SSqlObj* pSql = pObj->pSql; + if (pSql) { terrno = pSql->res.code; + sem_destroy(&pSql->rspSem); } taosTmrStopA(&(pObj->pTimer)); tscFreeSqlObj(pSql); - if (pSql) { - sem_destroy(&pSql->rspSem); - } rpcClose(pObj->pMgmtConn); - pthread_mutex_destroy(&pObj->mutex); tscTrace("%p DB connection is closed", pObj); tfree(pObj); } -bool tscIsInsertOrImportData(char* sqlstr) { +bool tscIsInsertData(char* sqlstr) { int32_t index = 0; do { @@ -843,12 +805,12 @@ SFieldSupInfo* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FI void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo) { size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); - SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); pExpr->offset = 0; for (int32_t i = 1; i < numOfExprs; ++i) { - SSqlExpr* prev = taosArrayGetP(pQueryInfo->exprsInfo, i - 1); - SSqlExpr* p = taosArrayGetP(pQueryInfo->exprsInfo, i); + SSqlExpr* prev = taosArrayGetP(pQueryInfo->exprList, i - 1); + SSqlExpr* p = taosArrayGetP(pQueryInfo->exprList, i); p->offset = prev->offset + prev->resBytes; } @@ -859,13 +821,13 @@ void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo) { return; } - SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); pExpr->offset = 0; size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); for (int32_t i = 1; i < numOfExprs; ++i) { - SSqlExpr* prev = taosArrayGetP(pQueryInfo->exprsInfo, i - 1); - SSqlExpr* p = taosArrayGetP(pQueryInfo->exprsInfo, i); + SSqlExpr* prev = taosArrayGetP(pQueryInfo->exprList, i - 1); + SSqlExpr* p = taosArrayGetP(pQueryInfo->exprList, i); p->offset = prev->offset + prev->resBytes; } @@ -874,8 +836,17 @@ void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo) { void tscFieldInfoCopy(SFieldInfo* dst, const SFieldInfo* src) { dst->numOfOutput = src->numOfOutput; - taosArrayCopy(dst->pFields, src->pFields); - taosArrayCopy(dst->pSupportInfo, src->pSupportInfo); + if (dst->pFields == NULL) { + dst->pFields = taosArrayClone(src->pFields); + } else { + taosArrayCopy(dst->pFields, src->pFields); + } + + if (dst->pSupportInfo == NULL) { + dst->pSupportInfo = taosArrayClone(src->pSupportInfo); + } else { + taosArrayCopy(dst->pSupportInfo, src->pSupportInfo); + } } TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index) { @@ -951,7 +922,6 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pColIndex->tableIndex); SSqlExpr* pExpr = calloc(1, sizeof(SSqlExpr)); - pExpr->functionId = functionId; // set the correct column index @@ -982,20 +952,20 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type, int16_t size, int16_t interSize, bool isTagCol) { - int32_t num = taosArrayGetSize(pQueryInfo->exprsInfo); + int32_t num = taosArrayGetSize(pQueryInfo->exprList); if (index == num) { return tscSqlExprAppend(pQueryInfo, functionId, pColIndex, type, size, interSize, isTagCol); } SSqlExpr* pExpr = doBuildSqlExpr(pQueryInfo, functionId, pColIndex, type, size, interSize, isTagCol); - taosArrayInsert(pQueryInfo->exprsInfo, index, &pExpr); + taosArrayInsert(pQueryInfo->exprList, index, &pExpr); return pExpr; } SSqlExpr* tscSqlExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, int16_t size, int16_t interSize, bool isTagCol) { SSqlExpr* pExpr = doBuildSqlExpr(pQueryInfo, functionId, pColIndex, type, size, interSize, isTagCol); - taosArrayPush(pQueryInfo->exprsInfo, &pExpr); + taosArrayPush(pQueryInfo->exprList, &pExpr); return pExpr; } @@ -1019,7 +989,7 @@ SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functi } int32_t tscSqlExprNumOfExprs(SQueryInfo* pQueryInfo) { - return taosArrayGetSize(pQueryInfo->exprsInfo); + return taosArrayGetSize(pQueryInfo->exprList); } void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes, int16_t tableIndex) { @@ -1036,7 +1006,7 @@ void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes, } SSqlExpr* tscSqlExprGet(SQueryInfo* pQueryInfo, int32_t index) { - return taosArrayGetP(pQueryInfo->exprsInfo, index); + return taosArrayGetP(pQueryInfo->exprList, index); } void* sqlExprDestroy(SSqlExpr* pExpr) { @@ -1067,14 +1037,10 @@ void tscSqlExprInfoDestroy(SArray* pExprInfo) { taosArrayDestroy(pExprInfo); } -SArray* tscSqlExprCopy(const SArray* src, uint64_t uid, bool deepcopy) { - if (src == NULL || taosArrayGetSize(src) == 0) { - return taosArrayInit(1, POINTER_BYTES); - } +void tscSqlExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy) { + assert(src != NULL && dst != NULL); size_t size = taosArrayGetSize(src); - SArray* dst = taosArrayInit(size, POINTER_BYTES); - for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = taosArrayGetP(src, i); @@ -1094,8 +1060,6 @@ SArray* tscSqlExprCopy(const SArray* src, uint64_t uid, bool deepcopy) { } } } - - return dst; } SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) { @@ -1193,9 +1157,7 @@ static void tscColumnDestroy(SColumn* pCol) { } void tscColumnListCopy(SArray* dst, const SArray* src, int16_t tableIndex) { - if (src == NULL) { - return; - } + assert(src != NULL && dst != NULL); size_t num = taosArrayGetSize(src); for (int32_t i = 0; i < num; ++i) { @@ -1576,9 +1538,22 @@ STableMetaInfo* tscGetTableMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, i return tscGetMetaInfo(pQueryInfo, k); } +void tscInitQueryInfo(SQueryInfo* pQueryInfo) { + assert(pQueryInfo->fieldsInfo.pFields == NULL); + pQueryInfo->fieldsInfo.pFields = taosArrayInit(4, sizeof(TAOS_FIELD)); + + assert(pQueryInfo->fieldsInfo.pSupportInfo == NULL); + pQueryInfo->fieldsInfo.pSupportInfo = taosArrayInit(4, sizeof(SFieldSupInfo)); + + assert(pQueryInfo->exprList == NULL); + pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES); + pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); +} + int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { assert(pCmd != NULL); + // todo refactor: remove this structure size_t s = pCmd->numOfClause + 1; char* tmp = realloc(pCmd->pQueryInfo, s * POINTER_BYTES); if (tmp == NULL) { @@ -1588,12 +1563,7 @@ int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { pCmd->pQueryInfo = (SQueryInfo**)tmp; SQueryInfo* pQueryInfo = calloc(1, sizeof(SQueryInfo)); - - // todo refactor to extract functions. - pQueryInfo->fieldsInfo.pFields = taosArrayInit(4, sizeof(TAOS_FIELD)); - pQueryInfo->fieldsInfo.pSupportInfo = taosArrayInit(4, sizeof(SFieldSupInfo)); - - pQueryInfo->exprsInfo = taosArrayInit(4, POINTER_BYTES); + tscInitQueryInfo(pQueryInfo); pQueryInfo->msg = pCmd->payload; // pointer to the parent error message buffer @@ -1605,11 +1575,11 @@ static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) { tscTagCondRelease(&pQueryInfo->tagCond); tscFieldInfoClear(&pQueryInfo->fieldsInfo); - tscSqlExprInfoDestroy(pQueryInfo->exprsInfo); - memset(&pQueryInfo->exprsInfo, 0, sizeof(pQueryInfo->exprsInfo)); + tscSqlExprInfoDestroy(pQueryInfo->exprList); + pQueryInfo->exprList = NULL; tscColumnListDestroy(pQueryInfo->colList); - memset(&pQueryInfo->colList, 0, sizeof(pQueryInfo->colList)); + pQueryInfo->colList = NULL; if (pQueryInfo->groupbyExpr.columnInfo != NULL) { taosArrayDestroy(pQueryInfo->groupbyExpr.columnInfo); @@ -1628,6 +1598,35 @@ void tscClearSubqueryInfo(SSqlCmd* pCmd) { } } +void doRemoveTableMetaInfo(SQueryInfo* pQueryInfo, int32_t index, bool removeFromCache) { + if (index < 0 || index >= pQueryInfo->numOfTables) { + return; + } + + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index); + + tscClearTableMetaInfo(pTableMetaInfo, removeFromCache); + free(pTableMetaInfo); + + int32_t after = pQueryInfo->numOfTables - index - 1; + if (after > 0) { + memmove(&pQueryInfo->pTableMetaInfo[index], &pQueryInfo->pTableMetaInfo[index + 1], after * POINTER_BYTES); + } + + pQueryInfo->numOfTables -= 1; +} + +void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache) { + tscTrace("%p deref the table meta in cache, numOfTables:%d", address, pQueryInfo->numOfTables); + + int32_t index = pQueryInfo->numOfTables; + while (index >= 0) { + doRemoveTableMetaInfo(pQueryInfo, --index, removeFromCache); + } + + tfree(pQueryInfo->pTableMetaInfo); +} + void tscFreeQueryInfo(SSqlCmd* pCmd) { if (pCmd == NULL || pCmd->numOfClause == 0) { return; @@ -1638,7 +1637,7 @@ void tscFreeQueryInfo(SSqlCmd* pCmd) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, i); freeQueryInfoImpl(pQueryInfo); - tscClearAllTableMetaInfo(pQueryInfo, (const char*)addr, false); + clearAllTableMetaInfo(pQueryInfo, (const char*)addr, false); tfree(pQueryInfo); } @@ -1689,35 +1688,6 @@ STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo* pQueryInfo) { return tscAddTableMetaInfo(pQueryInfo, NULL, NULL, NULL, NULL); } -void doRemoveTableMetaInfo(SQueryInfo* pQueryInfo, int32_t index, bool removeFromCache) { - if (index < 0 || index >= pQueryInfo->numOfTables) { - return; - } - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index); - - tscClearTableMetaInfo(pTableMetaInfo, removeFromCache); - free(pTableMetaInfo); - - int32_t after = pQueryInfo->numOfTables - index - 1; - if (after > 0) { - memmove(&pQueryInfo->pTableMetaInfo[index], &pQueryInfo->pTableMetaInfo[index + 1], after * POINTER_BYTES); - } - - pQueryInfo->numOfTables -= 1; -} - -void tscClearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache) { - tscTrace("%p deref the table meta in cache, numOfTables:%d", address, pQueryInfo->numOfTables); - - int32_t index = pQueryInfo->numOfTables; - while (index >= 0) { - doRemoveTableMetaInfo(pQueryInfo, --index, removeFromCache); - } - - tfree(pQueryInfo->pTableMetaInfo); -} - void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) { if (pTableMetaInfo == NULL) { return; @@ -1782,19 +1752,25 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void SQueryInfo* pNewQueryInfo = tscGetQueryInfoDetail(pnCmd, 0); SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - memcpy(pNewQueryInfo, pQueryInfo, sizeof(SQueryInfo)); - - memset(&pNewQueryInfo->fieldsInfo, 0, sizeof(SFieldInfo)); - + pNewQueryInfo->command = pQueryInfo->command; + pNewQueryInfo->slidingTimeUnit = pQueryInfo->slidingTimeUnit; + pNewQueryInfo->intervalTime = pQueryInfo->intervalTime; + pNewQueryInfo->slidingTime = pQueryInfo->slidingTime; + pNewQueryInfo->type = pQueryInfo->type; + pNewQueryInfo->window = pQueryInfo->window; + pNewQueryInfo->limit = pQueryInfo->limit; + pNewQueryInfo->slimit = pQueryInfo->slimit; + pNewQueryInfo->order = pQueryInfo->order; + pNewQueryInfo->clauseLimit = pQueryInfo->clauseLimit; pNewQueryInfo->pTableMetaInfo = NULL; - pNewQueryInfo->defaultVal = NULL; + pNewQueryInfo->defaultVal = NULL; pNewQueryInfo->numOfTables = 0; pNewQueryInfo->tsBuf = NULL; - pNewQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); - pNewQueryInfo->fieldsInfo.pFields = taosArrayInit(4, sizeof(TAOS_FIELD)); - pNewQueryInfo->fieldsInfo.pSupportInfo = taosArrayInit(4, sizeof(SFieldSupInfo)); - pNewQueryInfo->exprsInfo = taosArrayInit(4, POINTER_BYTES); + pNewQueryInfo->groupbyExpr = pQueryInfo->groupbyExpr; + if (pQueryInfo->groupbyExpr.columnInfo != NULL) { + pNewQueryInfo->groupbyExpr.columnInfo = taosArrayClone(pQueryInfo->groupbyExpr.columnInfo); + } tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond); @@ -1820,7 +1796,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void } uint64_t uid = pTableMetaInfo->pTableMeta->uid; - pNewQueryInfo->exprsInfo = tscSqlExprCopy(pQueryInfo->exprsInfo, uid, true); + tscSqlExprCopy(pNewQueryInfo->exprList, pQueryInfo->exprList, uid, true); int32_t numOfOutput = tscSqlExprNumOfExprs(pNewQueryInfo); @@ -1840,20 +1816,23 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void } } - // make sure the the sqlExpr for each fields is correct + // make sure the the sqlExpr for each fields is correct // todo handle the agg arithmetic expression for(int32_t f = 0; f < pNewQueryInfo->fieldsInfo.numOfOutput; ++f) { TAOS_FIELD* field = tscFieldInfoGetField(&pNewQueryInfo->fieldsInfo, f); + numOfExprs = tscSqlExprNumOfExprs(pNewQueryInfo); + for(int32_t k1 = 0; k1 < numOfExprs; ++k1) { SSqlExpr* pExpr1 = tscSqlExprGet(pNewQueryInfo, k1); - if (strcmp(field->name, pExpr1->aliasName) == 0) { + + if (strcmp(field->name, pExpr1->aliasName) == 0) { // eatablish link according to the result field name SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pNewQueryInfo->fieldsInfo, f); pInfo->pSqlExpr = pExpr1; } } } - - tscFieldInfoUpdateOffsetForInterResult(pNewQueryInfo); + + tscFieldInfoUpdateOffset(pNewQueryInfo); } pNew->fp = fp; @@ -1885,7 +1864,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void size_t size = taosArrayGetSize(pNewQueryInfo->colList); tscTrace( - "%p new subquery: %p, tableIndex:%d, vnodeIdx:%d, type:%d, exprInfo:%d, colList:%d," + "%p new subquery:%p, tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%d, colList:%d," "fieldInfo:%d, name:%s, qrang:%" PRId64 " - %" PRId64 " order:%d, limit:%" PRId64, pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscSqlExprNumOfExprs(pNewQueryInfo), size, pNewQueryInfo->fieldsInfo.numOfOutput, pFinalInfo->name, pNewQueryInfo->window.skey, @@ -1899,24 +1878,54 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void return pNew; } +/** + * To decide if current is a two-stage super table query, join query, or insert. And invoke different + * procedure accordingly + * @param pSql + */ void tscDoQuery(SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; + SSqlRes* pRes = &pSql->res; - pSql->res.code = TSDB_CODE_SUCCESS; + pRes->code = TSDB_CODE_SUCCESS; if (pCmd->command > TSDB_SQL_LOCAL) { tscProcessLocalCmd(pSql); + return; + } + + if (pCmd->command == TSDB_SQL_SELECT) { + tscAddIntoSqlList(pSql); + } + + if (pCmd->dataSourceType == DATA_FROM_DATA_FILE) { + tscProcessMultiVnodesInsertFromFile(pSql); } else { - if (pCmd->command == TSDB_SQL_SELECT) { - tscAddIntoSqlList(pSql); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + uint16_t type = pQueryInfo->type; + + if (pSql->fp == (void(*)())tscHandleMultivnodeInsert) { // multi-vnodes insertion + tscHandleMultivnodeInsert(pSql); + return; } - - if (pCmd->dataSourceType == DATA_FROM_DATA_FILE) { - tscProcessMultiVnodesInsertFromFile(pSql); - } else { - // pSql may be released in this function if it is a async insertion. - tscProcessSql(pSql); + + if (QUERY_IS_JOIN_QUERY(type)) { + if ((pQueryInfo->type & TSDB_QUERY_TYPE_SUBQUERY) == 0) { + tscHandleMasterJoinQuery(pSql); + return; + } else { + // for first stage sub query, iterate all vnodes to get all timestamp + if ((pQueryInfo->type & TSDB_QUERY_TYPE_JOIN_SEC_STAGE) != TSDB_QUERY_TYPE_JOIN_SEC_STAGE) { +// doProcessSql(pSql); + assert(0); + } + } + } else if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { // super table query + tscHandleMasterSTableQuery(pSql); + return; } + + tscProcessSql(pSql); } } @@ -2095,7 +2104,7 @@ void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)()) { //backup the total number of result first int64_t num = pRes->numOfTotal + pRes->numOfTotalInCurrentClause; - tscFreeResData(pSql); + tscFreeSqlResult(pSql); pRes->numOfTotal = num; @@ -2115,10 +2124,22 @@ void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)()) { } } -char* tscGetResultColumnChr(SSqlRes* pRes, SQueryInfo* pQueryInfo, int32_t column) { +char* tscGetResultColumnChr(SSqlRes* pRes, SQueryInfo* pQueryInfo, int32_t column, int16_t bytes) { SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, column); - return ((char*) pRes->data) + pInfo->pSqlExpr->offset * pRes->numOfRows; + int32_t type = pInfo->pSqlExpr->resType; + char* pData = ((char*) pRes->data) + pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row; + + if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { + int32_t realLen = varDataLen(pData); + if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor + *(char*) (pData + realLen + sizeof(int16_t)) = 0; + } + + return pData + VARSTR_HEADER_SIZE; // head is the length of binary/nchar data + } else { + return pData; + } } diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index d655bdcfe3b82fe0b97fff29fb5ec847bd97955e..78b6cb73b2bc0e960fc4ec963a55d82e2d70c852 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -26,6 +26,20 @@ extern "C" { #endif +#define STR_TO_VARSTR(x, str) do {VarDataLenT __len = strlen(str); \ + *(VarDataLenT*)(x) = __len; \ + strncpy((char*)(x) + VARSTR_HEADER_SIZE, (str), __len);} while(0); + +#define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs) do {\ + char* _e = stpncpy((char*)(x) + VARSTR_HEADER_SIZE, (str), (_maxs));\ + *(VarDataLenT*)(x) = _e - (x);\ +} while(0) + +#define STR_WITH_SIZE_TO_VARSTR(x, str, _size) do {\ + *(VarDataLenT*)(x) = (_size); \ + strncpy((char*)(x) + VARSTR_HEADER_SIZE, (str), (_size));\ +} while(0); + // ----------------- TSDB COLUMN DEFINITION typedef struct { int8_t type; // Column type @@ -182,7 +196,7 @@ void tdInitDataCols(SDataCols *pCols, STSchema *pSchema); SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData); void tdFreeDataCols(SDataCols *pCols); void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols); -void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop); +void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop); //!!!! int tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge); void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, SDataCols *src2, int *iter2, int tRows); diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 7321e1c921bab082f3074b087cab12b30da00c4c..02351f7d3aba514628305370b23734b0039ff5e8 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -172,28 +172,9 @@ int tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_ switch (type) { case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: - // set offset *(VarDataOffsetT *)POINTER_DRIFT(row, toffset) = dataRowLen(row); - - // set length - VarDataLenT slen = 0; - if (isNull(value, type)) { - slen = (type == TSDB_DATA_TYPE_BINARY) ? sizeof(int8_t) : sizeof(int32_t); - } else { - if (type == TSDB_DATA_TYPE_BINARY) { - slen = strnlen((char *)value, bytes); - } else { - slen = wcsnlen((wchar_t *)value, (bytes) / TSDB_NCHAR_SIZE) * TSDB_NCHAR_SIZE; - } - } - - ASSERT(slen <= bytes); - *(VarDataLenT *)ptr = slen; - ptr = POINTER_DRIFT(ptr, sizeof(VarDataLenT)); - - memcpy((void *)ptr, value, slen); - dataRowLen(row) += (sizeof(int16_t) + slen); - + memcpy(ptr, value, varDataTLen(value)); + dataRowLen(row) += varDataTLen(value); break; default: memcpy(POINTER_DRIFT(row, toffset), value, TYPE_BYTES[type]); diff --git a/src/common/src/tname.c b/src/common/src/tname.c index 822d612243dda73ad98c4f5d7ffbdba017e17f71..29236ed0ff89a4bb2b803a4d924591ba7a05c291 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -25,13 +25,10 @@ __attribute__((unused)) static FORCE_INLINE size_t copy(char* dst, const char* s } void extractTableName(const char* tableId, char* name) { - size_t offset = strcspn(tableId, &TS_PATH_DELIMITER[0]); - offset = strcspn(&tableId[offset], &TS_PATH_DELIMITER[0]); + size_t s1 = strcspn(tableId, &TS_PATH_DELIMITER[0]); + size_t s2 = strcspn(&tableId[s1 + 1], &TS_PATH_DELIMITER[0]); - strncpy(name, &tableId[offset], TSDB_TABLE_NAME_LEN); - -// char* r = skipSegments(tableId, TS_PATH_DELIMITER[0], 2); -// return copy(name, r, TS_PATH_DELIMITER[0]); + strncpy(name, &tableId[s1 + s2 + 2], TSDB_TABLE_NAME_LEN); } char* extractDBName(const char* tableId, char* name) { diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c index d99e916c73974b81ac8e8f08a305445ef76e31e0..533f992151f6d829e7f941d466c38e0f3ef47512 100644 --- a/src/common/src/ttypes.c +++ b/src/common/src/ttypes.c @@ -198,7 +198,7 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) { break; }; case TSDB_DATA_TYPE_BINARY: { - strncpy(val, src, len); + varDataCopy(val, src); break; }; case TSDB_DATA_TYPE_NCHAR: { diff --git a/src/dnode/src/dnodeWrite.c b/src/dnode/src/dnodeWrite.c index 39757c690f6d460fab2378c249f9ce9786ac3e22..babbcf4ae8aa2bcaa1f6f3dc647c5cede2149567 100644 --- a/src/dnode/src/dnodeWrite.c +++ b/src/dnode/src/dnodeWrite.c @@ -228,7 +228,7 @@ static void dnodeHandleIdleWorker(SWriteWorker *pWorker) { int32_t num = taosGetQueueNumber(pWorker->qset); if (num > 0) { - usleep(30000); + usleep(30); sched_yield(); } else { taosFreeQall(pWorker->qall); diff --git a/src/inc/query.h b/src/inc/query.h index ffeb225223e8276ca1a4d0a57a561d379dee578e..cdadd4759fcfd2b8ae5b1ba8a431dbb56bfac08c 100644 --- a/src/inc/query.h +++ b/src/inc/query.h @@ -23,12 +23,12 @@ typedef void* qinfo_t; /** * create the qinfo object according to QueryTableMsg - * @param pVnode + * @param tsdb * @param pQueryTableMsg * @param qinfo * @return */ -int32_t qCreateQueryInfo(void* pVnode, SQueryTableMsg* pQueryTableMsg, qinfo_t* qinfo); +int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, qinfo_t* qinfo); /** * Destroy QInfo object diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 1142f02922850e65e1bd85436ef46adb48219cc8..aa4889ec9d7e22c15b58420356096351327c2cc8 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -35,12 +35,15 @@ extern "C" { // ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR typedef int32_t VarDataOffsetT; typedef int16_t VarDataLenT; -#define varDataLen(v) ((VarDataLenT *)(v))[0] -#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v)) -#define varDataVal(v) ((void *)((char *)v + sizeof(VarDataLenT))) + +#define varDataLen(v) ((VarDataLenT *)(v))[0] +#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v)) +#define varDataVal(v) ((void *)((char *)v + sizeof(VarDataLenT))) +#define varDataCopy(dst, v) memcpy((dst), (void*) (v), varDataTLen(v)) // this data type is internally used only in 'in' query to hold the values #define TSDB_DATA_TYPE_ARRAY (TSDB_DATA_TYPE_NCHAR + 1) +#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) // Bytes for each type. extern const int32_t TYPE_BYTES[11]; @@ -209,7 +212,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_SHELL_VNODE_BITS 24 #define TSDB_SHELL_SID_MASK 0xFF #define TSDB_HTTP_TOKEN_LEN 20 -#define TSDB_SHOW_SQL_LEN 32 +#define TSDB_SHOW_SQL_LEN 512 #define TSDB_METER_STATE_OFFLINE 0 #define TSDB_METER_STATE_ONLLINE 1 @@ -290,27 +293,30 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_MAX_RPC_THREADS 5 -#define TSDB_QUERY_TYPE_NON_TYPE 0x00U // none type -#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01U // free qhandle at vnode +#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type +#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode /* * 1. ordinary sub query for select * from super_table * 2. all sqlobj generated by createSubqueryObj with this flag */ -#define TSDB_QUERY_TYPE_SUBQUERY 0x02U -#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04U // two-stage subquery for super table +#define TSDB_QUERY_TYPE_SUBQUERY 0x02u +#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04u // two-stage subquery for super table -#define TSDB_QUERY_TYPE_TABLE_QUERY 0x08U // query ordinary table; below only apply to client side -#define TSDB_QUERY_TYPE_STABLE_QUERY 0x10U // query on super table -#define TSDB_QUERY_TYPE_JOIN_QUERY 0x20U // join query -#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40U // select *,columns... query -#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE 0x80U // join sub query at the second stage +#define TSDB_QUERY_TYPE_TABLE_QUERY 0x08u // query ordinary table; below only apply to client side +#define TSDB_QUERY_TYPE_STABLE_QUERY 0x10u // query on super table +#define TSDB_QUERY_TYPE_JOIN_QUERY 0x20u // join query +#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40u // select *,columns... query +#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE 0x80u // join sub query at the second stage -#define TSDB_QUERY_TYPE_INSERT 0x100U // insert type -#define TSDB_QUERY_TYPE_IMPORT 0x200U // import data +#define TSDB_QUERY_TYPE_TAG_FILTER_QUERY 0x400u +#define TSDB_QUERY_TYPE_INSERT 0x100u // insert type +#define TSDB_QUERY_TYPE_IMPORT 0x200u // import data +#define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x800u #define TSDB_QUERY_HAS_TYPE(x, _type) (((x) & (_type)) != 0) #define TSDB_QUERY_SET_TYPE(x, _type) ((x) |= (_type)) +#define TSDB_QUERY_CLEAR_TYPE(x, _type) ((x) &= (~_type)) #define TSDB_QUERY_RESET_TYPE(x) ((x) = TSDB_QUERY_TYPE_NON_TYPE) #define TSDB_ORDER_ASC 1 diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index fd3105e3bb9a6ed0189816ec9dc463d4b4d09337..68e72b0964da9106bb8cce6079a221bb077482c7 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -380,13 +380,13 @@ typedef struct SSqlFuncMsg { } arg[3]; } SSqlFuncMsg; -typedef struct SArithExprInfo { +typedef struct SExprInfo { SSqlFuncMsg base; struct tExprNode* pExpr; int16_t bytes; int16_t type; int16_t interResBytes; -} SArithExprInfo; +} SExprInfo; typedef struct SColumnFilterInfo { int16_t lowerRelOptr; @@ -422,8 +422,8 @@ typedef struct SColumnInfo { } SColumnInfo; typedef struct STableIdInfo { - int32_t sid; int64_t uid; + int32_t tid; TSKEY key; // last accessed ts, for subscription } STableIdInfo; @@ -459,9 +459,6 @@ typedef struct { int16_t tagNameRelType; // relation of tag criteria and tbname criteria int16_t interpoType; // interpolate type uint64_t defaultVal; // default value array list - -// int32_t colNameLen; -// int64_t colNameList; int32_t tsOffset; // offset value in current msg body, NOTE: ts list is compressed int32_t tsLen; // total length of ts comp block int32_t tsNumOfBlocks; // ts comp block numbers @@ -624,8 +621,8 @@ typedef struct { } SCMMultiTableInfoMsg; typedef struct SCMSTableVgroupMsg { - char tableId[TSDB_TABLE_ID_LEN]; -} SCMSTableVgroupMsg; + int32_t numOfTables; +} SCMSTableVgroupMsg, SCMSTableVgroupRspMsg; typedef struct { int32_t vgId; @@ -634,59 +631,16 @@ typedef struct { } SCMVgroupInfo; typedef struct { - int32_t numOfVgroups; + int32_t numOfVgroups; SCMVgroupInfo vgroups[]; -} SCMSTableVgroupRspMsg; - -typedef struct { - int16_t elemLen; - - char tableId[TSDB_TABLE_ID_LEN + 1]; - int16_t orderIndex; - int16_t orderType; // used in group by xx order by xxx - - int16_t rel; // denotes the relation between condition and table list - - int32_t tableCond; // offset value of table name condition - int32_t tableCondLen; - - int32_t cond; // offset of column query condition - int32_t condLen; - - int16_t tagCols[TSDB_MAX_TAGS + 1]; // required tag columns, plus one is for table name - int16_t numOfTags; // required number of tags - - int16_t numOfGroupCols; // num of group by columns - int32_t groupbyTagColumnList; -} SSuperTableMetaElemMsg; - -typedef struct { - int32_t numOfTables; - int32_t join; - int32_t joinCondLen; // for join condition - int32_t metaElem[TSDB_MAX_JOIN_TABLE_NUM]; -} SSuperTableMetaMsg; - -typedef struct { - int32_t nodeId; - uint32_t nodeIp; - uint16_t nodePort; -} SVnodeDesc; - -typedef struct { - SVnodeDesc vpeerDesc[TSDB_MAX_REPLICA_NUM]; - int16_t index; // used locally - int32_t vgId; - int32_t numOfSids; - int32_t pSidExtInfoList[]; // offset value of STableIdInfo -} SVnodeSidList; - -typedef struct { - int32_t numOfTables; - int32_t numOfVnodes; - uint16_t tagLen; /* tag value length */ - int32_t list[]; /* offset of SVnodeSidList, compared to the SSuperTableMeta struct */ -} SSuperTableMeta; +} SVgroupsInfo; + +//typedef struct { +// int32_t numOfTables; +// int32_t join; +// int32_t joinCondLen; // for join condition +// int32_t metaElem[TSDB_MAX_JOIN_TABLE_NUM]; +//} SSuperTableMetaMsg; typedef struct STableMetaMsg { int32_t contLen; diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 35cf1b52c756d857ceffc0be5d4bdc7c50a3e30a..f4242fbe1fbb72fbbf1e189100c6d9e0fd3d4676 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -101,8 +101,8 @@ int tsdbTableSetName(STableCfg *config, char *name, bool dup); int tsdbTableSetSName(STableCfg *config, char *sname, bool dup); void tsdbClearTableCfg(STableCfg *config); -int32_t tsdbGetTableTagVal(TsdbRepoT *repo, STableId id, int32_t col, int16_t *type, int16_t *bytes, char **val); -int32_t tsdbTableGetName(TsdbRepoT *repo, STableId id, char** name); +int32_t tsdbGetTableTagVal(TsdbRepoT *repo, STableId* id, int32_t colId, int16_t *type, int16_t *bytes, char **val); +int32_t tsdbGetTableName(TsdbRepoT *repo, STableId* id, char** name); int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg); int tsdbDropTable(TsdbRepoT *pRepo, STableId tableId); @@ -142,34 +142,20 @@ int32_t tsdbInsertData(TsdbRepoT *pRepo, SSubmitMsg *pMsg); typedef void *TsdbQueryHandleT; // Use void to hide implementation details -typedef struct STableGroupList { // qualified table object list in group - SArray *pGroupList; - int32_t numOfTables; -} STableGroupList; - // query condition to build vnode iterator typedef struct STsdbQueryCond { STimeWindow twindow; - int32_t order; // desc/asc order to iterate the data block + int32_t order; // desc|asc order to iterate the data block int32_t numOfCols; SColumnInfo *colList; } STsdbQueryCond; -typedef struct SBlockInfo { - STimeWindow window; - - int32_t numOfRows; - int32_t numOfCols; - - STableId tableId; -} SBlockInfo; - typedef struct SDataBlockInfo { STimeWindow window; int32_t rows; int32_t numOfCols; int64_t uid; - int32_t sid; + int32_t tid; } SDataBlockInfo; typedef struct { @@ -192,14 +178,29 @@ typedef void *TsdbPosT; /** * Get the data block iterator, starting from position according to the query condition - * @param pCond query condition, only includes the filter on primary time stamp - * @param pTableList table sid list + * + * @param tsdb tsdb handle + * @param pCond query condition, including time window, result set order, and basic required columns for each block + * @param groupInfo tableId list in the form of set, seperated into different groups according to group by condition * @return */ TsdbQueryHandleT *tsdbQueryTables(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo); /** - * move to next block + * Get the last row of the given query time window for all the tables in STableGroupInfo object. + * Note that only one data block with only row will be returned while invoking retrieve data block function for + * all tables in this group. + * + * @param tsdb tsdb handle + * @param pCond query condition, including time window, result set order, and basic required columns for each block + * @param groupInfo tableId list. + * @return + */ +TsdbQueryHandleT tsdbQueryLastRow(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo); + +/** + * move to next block if exists + * * @param pQueryHandle * @return */ @@ -226,26 +227,16 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle); int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT *pQueryHandle, SDataStatis **pBlockStatis); /** + * * The query condition with primary timestamp is passed to iterator during its constructor function, * the returned data block must be satisfied with the time window condition in any cases, * which means the SData data block is not actually the completed disk data blocks. * - * @param pQueryHandle - * @return - */ -SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pQueryHandle, SArray *pIdList); - -/** - * todo remove the parameter of position, and order type - * - * Reset to the start(end) position of current query, from which the iterator starts. - * - * @param pQueryHandle - * @param position set the iterator traverses position - * @param order ascending order or descending order + * @param pQueryHandle query handle + * @param pColumnIdList required data columns id list * @return */ -int32_t tsdbResetQuery(TsdbQueryHandleT *pQueryHandle, STimeWindow *window, TsdbPosT position, int16_t order); +SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pQueryHandle, SArray *pColumnIdList); /** * todo remove this function later @@ -282,20 +273,19 @@ SArray *tsdbGetTableList(TsdbQueryHandleT *pQueryHandle); * Get the qualified table id for a super table according to the tag query expression. * @param stableid. super table sid * @param pTagCond. tag query condition - * */ -int32_t tsdbQueryByTagsCond( - TsdbRepoT *tsdb, - int64_t uid, - const char *pTagCond, - size_t len, - int16_t tagNameRelType, - const char* tbnameCond, - STableGroupInfo *pGroupList, - SColIndex *pColIndex, - int32_t numOfCols - ); +int32_t tsdbQuerySTableByTagCond(TsdbRepoT *tsdb, int64_t uid, const char *pTagCond, size_t len, + int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo *pGroupList, SColIndex *pColIndex, int32_t numOfCols); + +/** + * create the table group result including only one table, used to handle the normal table query + * + * @param tsdb tsdbHandle + * @param uid table uid + * @param pGroupInfo the generated result + * @return + */ int32_t tsdbGetOneTableGroup(TsdbRepoT *tsdb, int64_t uid, STableGroupInfo *pGroupInfo); /** diff --git a/src/mnode/inc/mgmtDef.h b/src/mnode/inc/mgmtDef.h index f6a85ec237751dda77a46a97abc9edb9c3861c7e..8e5240910d4300e757590f3b607bd888416fc332 100644 --- a/src/mnode/inc/mgmtDef.h +++ b/src/mnode/inc/mgmtDef.h @@ -66,6 +66,7 @@ typedef struct SMnodeObj { SDnodeObj *pDnode; } SMnodeObj; +// todo use dynamic length string typedef struct { char tableId[TSDB_TABLE_ID_LEN + 1]; int8_t type; diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index a904f3e71a0f3a0e696f63b89adeae15a9234395..9cc7e200d15a1d3786d9ad8f71b6e11493d1452e 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -23,6 +23,7 @@ #include "ttime.h" #include "tname.h" #include "tbalance.h" +#include "tdataformat.h" #include "mgmtDef.h" #include "mgmtLog.h" #include "mgmtAcct.h" @@ -431,7 +432,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) SUserObj *pUser = mgmtGetUserFromConn(pConn); if (pUser == NULL) return 0; - pShow->bytes[cols] = TSDB_DB_NAME_LEN; + pShow->bytes[cols] = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "name"); pSchema[cols].bytes = htons(pShow->bytes[cols]); @@ -439,7 +440,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create time"); + strcpy(pSchema[cols].name, "created_time"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -586,7 +587,9 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * cols = 0; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, mgmtGetDbStr(pDb->name)); + + char* name = mgmtGetDbStr(pDb->name); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, name, TSDB_DB_NAME_LEN); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; @@ -622,7 +625,10 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * #endif pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - sprintf(pWrite, "%d,%d,%d", pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2, pDb->cfg.daysToKeep); + + char tmp[128] = {0}; + size_t n = sprintf(tmp, "%d,%d,%d", pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2, pDb->cfg.daysToKeep); + STR_WITH_SIZE_TO_VARSTR(pWrite, tmp, n); cols++; #ifndef __CLOUD_VERSION__ @@ -670,7 +676,11 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, pDb->status != TSDB_DB_STATUS_READY ? "dropping" : "ready"); + if (pDb->status == TSDB_DB_STATUS_READY) { + STR_WITH_SIZE_TO_VARSTR(pWrite, "ready", 5); + } else { + STR_WITH_SIZE_TO_VARSTR(pWrite, "dropping", 8); + } cols++; numOfRows++; diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 6ed19b3d1146e8478ff7cc2f84121b55e5396433..53a208745c7c1e0e1906d10cc9a69d9c7a8ff6a9 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -40,6 +40,7 @@ #include "mgmtUser.h" #include "mgmtVgroup.h" #include "tcompare.h" +#include "tdataformat.h" static void * tsChildTableSdb; static void * tsSuperTableSdb; @@ -592,11 +593,11 @@ void *mgmtGetTable(char *tableId) { } void *mgmtGetNextChildTable(void *pNode, SChildTableObj **pTable) { - return sdbFetchRow(tsChildTableSdb, pNode, (void **)pTable); + return sdbFetchRow(tsChildTableSdb, pNode, (void **)pTable); } void *mgmtGetNextSuperTable(void *pNode, SSuperTableObj **pTable) { - return sdbFetchRow(tsSuperTableSdb, pNode, (void **)pTable); + return sdbFetchRow(tsSuperTableSdb, pNode, (void **)pTable); } void mgmtIncTableRef(void *p1) { @@ -624,6 +625,7 @@ void mgmtCleanUpTables() { mgmtCleanUpSuperTables(); } +// todo move to name.h, add length of table name static void mgmtExtractTableName(char* tableId, char* name) { int pos = -1; int num = 0; @@ -1056,7 +1058,7 @@ static int32_t mgmtGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, int32_t cols = 0; SSchema *pSchema = pMeta->schema; - pShow->bytes[cols] = TSDB_TABLE_NAME_LEN; + pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "name"); pSchema[cols].bytes = htons(pShow->bytes[cols]); @@ -1064,7 +1066,7 @@ static int32_t mgmtGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create_time"); + strcpy(pSchema[cols].name, "created_time"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -1136,7 +1138,12 @@ int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, v cols = 0; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strncpy(pWrite, stableName, TSDB_TABLE_NAME_LEN); + + int16_t len = strnlen(stableName, TSDB_DB_NAME_LEN); + *(int16_t*) pWrite = len; + pWrite += sizeof(int16_t); // todo refactor + + strncpy(pWrite, stableName, len); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; @@ -1233,46 +1240,66 @@ static void mgmtGetSuperTableMeta(SQueuedMsg *pMsg) { static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) { SCMSTableVgroupMsg *pInfo = pMsg->pCont; - if (pMsg->pTable == NULL) pMsg->pTable = mgmtGetSuperTable(pInfo->tableId); - if (pMsg->pTable == NULL) { - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); - return; - } - - SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable; - int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + sizeof(SCMVgroupInfo) * pTable->vgLen; - SCMSTableVgroupRspMsg *pRsp = rpcMallocCont(contLen); + int32_t numOfTable = htonl(pInfo->numOfTables); + + char* name = (char*) pInfo + sizeof(struct SCMSTableVgroupMsg); + SCMSTableVgroupRspMsg *pRsp = NULL; + + // todo set the initial size to be 10, fix me + int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + (sizeof(SCMVgroupInfo) * 10 + sizeof(SVgroupsInfo))*numOfTable; + + pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY); return; } - int32_t vg = 0; - for (; vg < pTable->vgLen; ++vg) { - int32_t vgId = pTable->vgList[vg]; - if (vgId == 0) break; - - SVgObj *pVgroup = mgmtGetVgroup(vgId); - if (pVgroup == NULL) break; - - pRsp->vgroups[vg].vgId = htonl(vgId); - for (int32_t vn = 0; vn < pVgroup->numOfVnodes; ++vn) { - SDnodeObj *pDnode = pVgroup->vnodeGid[vn].pDnode; - if (pDnode == NULL) break; - - strcpy(pRsp->vgroups[vg].ipAddr[vn].fqdn, pDnode->dnodeFqdn); - pRsp->vgroups[vg].ipAddr[vn].port = htons(pDnode->dnodePort + TSDB_PORT_DNODESHELL); - pRsp->vgroups[vg].numOfIps++; + pRsp->numOfTables = htonl(numOfTable); + char* msg = (char*) pRsp + sizeof(SCMSTableVgroupRspMsg); + + for(int32_t i = 0; i < numOfTable; ++i) { + SSuperTableObj *pTable = mgmtGetSuperTable(name); + + pMsg->pTable = (STableObj *)pTable; + if (pMsg->pTable == NULL) { + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); + return; + } + + SVgroupsInfo* pVgroup = (SVgroupsInfo*) msg; + + int32_t vg = 0; + for (; vg < pTable->vgLen; ++vg) { + int32_t vgId = pTable->vgList[vg]; + if (vgId == 0) break; + + SVgObj *vgItem = mgmtGetVgroup(vgId); + if (vgItem == NULL) break; + + pVgroup->vgroups[vg].vgId = htonl(vgId); + for (int32_t vn = 0; vn < vgItem->numOfVnodes; ++vn) { + SDnodeObj *pDnode = vgItem->vnodeGid[vn].pDnode; + if (pDnode == NULL) break; + + strncpy(pVgroup->vgroups[vg].ipAddr[vn].fqdn, pDnode->dnodeFqdn, tListLen(pDnode->dnodeFqdn)); + pVgroup->vgroups[vg].ipAddr[vn].port = htons(tsDnodeShellPort); + + pVgroup->vgroups[vg].numOfIps++; + } + + mgmtDecVgroupRef(vgItem); } - - mgmtDecVgroupRef(pVgroup); + + pVgroup->numOfVgroups = htonl(vg); + + // one table is done, try the next table + msg += sizeof(SVgroupsInfo) + vg * sizeof(SCMVgroupInfo); } - pRsp->numOfVgroups = htonl(vg); SRpcMsg rpcRsp = {0}; rpcRsp.handle = pMsg->thandle; rpcRsp.pCont = pRsp; - rpcRsp.contLen = sizeof(SCMSTableVgroupRspMsg) + sizeof(SCMVgroupInfo) * vg; + rpcRsp.contLen = msg - (char*) pRsp; rpcSendResponse(&rpcRsp); } @@ -1705,7 +1732,7 @@ static void mgmtAutoCreateChildTable(SQueuedMsg *pMsg) { mgmtAddToShellQueue(newMsg); } -static void mgmtGetChildTableMeta(SQueuedMsg *pMsg) { +static void mgmtGetChildTableMeta(SQueuedMsg *pMsg) { STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS); if (pMeta == NULL) { mError("table:%s, failed to get table meta, no enough memory", pMsg->pTable->tableId); @@ -1989,15 +2016,15 @@ static int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void int32_t cols = 0; SSchema *pSchema = pMeta->schema; - pShow->bytes[cols] = TSDB_TABLE_NAME_LEN; + pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "table name"); + strcpy(pSchema[cols].name, "table_name"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create time"); + strcpy(pSchema[cols].name, "created_time"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -2007,9 +2034,9 @@ static int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = TSDB_TABLE_NAME_LEN; + pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "stable name"); + strcpy(pSchema[cols].name, "stable_name"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -2072,7 +2099,8 @@ static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, int32_t cols = 0; char *pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strncpy(pWrite, tableName, TSDB_TABLE_NAME_LEN); + + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, tableName, TSDB_TABLE_NAME_LEN); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; @@ -2089,9 +2117,13 @@ static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + + memset(tableName, 0, tListLen(tableName)); if (pTable->info.type == TSDB_CHILD_TABLE) { - mgmtExtractTableName(pTable->superTable->info.tableId, pWrite); + mgmtExtractTableName(pTable->superTable->info.tableId, tableName); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, tableName, TSDB_TABLE_NAME_LEN); } + cols++; numOfRows++; diff --git a/src/query/inc/qast.h b/src/query/inc/qast.h index 6c997d5a36dd3ba379f0d490c7178c02b2340df0..8698b3af82cd46676cade40af1c7ad4f8a42f463 100644 --- a/src/query/inc/qast.h +++ b/src/query/inc/qast.h @@ -48,7 +48,8 @@ typedef struct tQueryInfo { int32_t colIndex; // index of column in schema uint8_t optr; // expression operator SSchema sch; // schema of tags - tVariant q; // query condition value on the specific schema, filter expression +// tVariant q; // query condition value on the specific schema, filter expression + char* q; __compar_fn_t compare; // filter function } tQueryInfo; diff --git a/src/query/inc/qresultBuf.h b/src/query/inc/qresultBuf.h index 39600f512963745a90bd082bdfc3c412d38374c7..2e813dbd98417eda0411d469653c8a1573b4be38 100644 --- a/src/query/inc/qresultBuf.h +++ b/src/query/inc/qresultBuf.h @@ -51,7 +51,7 @@ typedef struct SDiskbasedResultBuf { * @param rowSize * @return */ -int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t size, int32_t rowSize); +int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t size, int32_t rowSize, void* handle); /** * @@ -103,7 +103,7 @@ int32_t getNumOfResultBufGroupId(SDiskbasedResultBuf* pResultBuf); * destroy result buffer * @param pResultBuf */ -void destroyResultBuf(SDiskbasedResultBuf* pResultBuf); +void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle); /** * diff --git a/src/query/inc/qsqltype.h b/src/query/inc/qsqltype.h index ad9affa1ccdc3a7d729646ba1c0bf67063273ad1..08d30be925796654ec127de997e6a236ea39f6fc 100644 --- a/src/query/inc/qsqltype.h +++ b/src/query/inc/qsqltype.h @@ -60,17 +60,16 @@ enum _sql_type { TSDB_SQL_LOCAL, // SQL below for client local TSDB_SQL_DESCRIBE_TABLE, - TSDB_SQL_RETRIEVE_METRIC, + TSDB_SQL_RETRIEVE_LOCALMERGE, TSDB_SQL_METRIC_JOIN_RETRIEVE, - TSDB_SQL_RETRIEVE_TAGS, /* * build empty result instead of accessing dnode to fetch result * reset the client cache */ - TSDB_SQL_RETRIEVE_EMPTY_RESULT, // 40 + TSDB_SQL_RETRIEVE_EMPTY_RESULT, - TSDB_SQL_RESET_CACHE, + TSDB_SQL_RESET_CACHE, // 40 TSDB_SQL_SERV_STATUS, TSDB_SQL_CURRENT_DB, TSDB_SQL_SERV_VERSION, @@ -78,7 +77,7 @@ enum _sql_type { TSDB_SQL_CURRENT_USER, TSDB_SQL_CFG_LOCAL, - TSDB_SQL_MAX // 48 + TSDB_SQL_MAX // 47 }; diff --git a/src/query/inc/qtsbuf.h b/src/query/inc/qtsbuf.h index 8e014e5feb781b2c87209118d1c4c8a7c2d13813..c83c3dbe25d0ede36fbaedca357efa7082013710 100644 --- a/src/query/inc/qtsbuf.h +++ b/src/query/inc/qtsbuf.h @@ -48,7 +48,7 @@ typedef struct STSElem { } STSElem; typedef struct STSCursor { - int32_t vnodeIndex; + int32_t vgroupIndex; int32_t blockIndex; int32_t tsIndex; uint32_t order; diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index 13b0195893ba936c496bd16b36f383193a35a61b..906dadb31798b2a48b0ebb142c613511a49dee65 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -96,24 +96,20 @@ typedef struct SSingleColumnFilterInfo { void* pData; } SSingleColumnFilterInfo; -typedef struct STableQueryInfo { - int64_t lastKey; - STimeWindow win; +typedef struct STableQueryInfo { // todo merge with the STableQueryInfo struct + int32_t tableIndex; + int32_t groupIdx; // group id in table list + TSKEY lastKey; int32_t numOfRes; int16_t queryRangeSet; // denote if the query range is set, only available for interval query int64_t tag; + STimeWindow win; STSCursor cur; - int32_t tid; // for retrieve the page id list - + STableId id; // for retrieve the page id list + SWindowResInfo windowResInfo; } STableQueryInfo; -typedef struct STableDataInfo { // todo merge with the STableQueryInfo struct - int32_t tableIndex; - int32_t groupIdx; // group id in table list - STableQueryInfo* pTableQInfo; -} STableDataInfo; - typedef struct SQuery { int16_t numOfCols; int16_t numOfTags; @@ -130,7 +126,7 @@ typedef struct SQuery { SLimitVal limit; int32_t rowSize; SSqlGroupbyExpr* pGroupbyExpr; - SArithExprInfo* pSelectExpr; + SExprInfo* pSelectExpr; SColumnInfo* colList; SColumnInfo* tagColList; int32_t numOfFilterCols; @@ -170,14 +166,16 @@ typedef struct SQInfo { TSKEY startTime; TSKEY elapsedTime; int32_t pointsInterpo; - int32_t code; // error code to returned to client + int32_t code; // error code to returned to client sem_t dataReady; void* tsdb; + int32_t vgId; - STableGroupInfo groupInfo; // table id list + STableGroupInfo tableIdGroupInfo; // table id list < only includes the STableId list> + STableGroupInfo groupInfo; // SQueryRuntimeEnv runtimeEnv; int32_t groupIndex; - int32_t offset; /* offset in group result set of subgroup */ + int32_t offset; // offset in group result set of subgroup, todo refactor T_REF_DECLARE() /* @@ -188,7 +186,6 @@ typedef struct SQInfo { */ int32_t tableIndex; int32_t numOfGroupResultPages; - TSKEY* tsList; } SQInfo; #endif // TDENGINE_QUERYEXECUTOR_H diff --git a/src/query/inc/queryLog.h b/src/query/inc/queryLog.h index 929e079d1edb265cde81cce50c9699e1efd7e478..2c77ee6032afa9033f909d68a59d6658028f9988 100644 --- a/src/query/inc/queryLog.h +++ b/src/query/inc/queryLog.h @@ -31,11 +31,11 @@ extern int32_t qdebugFlag; #define qError(...) \ if (qdebugFlag & DEBUG_ERROR) { \ - taosPrintLog("ERROR RPC ", qdebugFlag, __VA_ARGS__); \ + taosPrintLog("ERROR QRY ", qdebugFlag, __VA_ARGS__); \ } #define qWarn(...) \ if (qdebugFlag & DEBUG_WARN) { \ - taosPrintLog("WARN RPC ", qdebugFlag, __VA_ARGS__); \ + taosPrintLog("WARN QRY ", qdebugFlag, __VA_ARGS__); \ } #ifdef __cplusplus diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index 29d9d8aaac90d543e3d93e61216d29b69dd61c20..be8d922a849f14b3be3c69807c0669581fe4f0b9 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -120,7 +120,7 @@ cmd ::= DROP TABLE ifexists(Y) ids(X) cpxName(Z). { } cmd ::= DROP DATABASE ifexists(Y) ids(X). { setDropDBTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y); } -cmd ::= DROP DNODE IPTOKEN(X). { setDCLSQLElems(pInfo, TSDB_SQL_DROP_DNODE, 1, &X); } +cmd ::= DROP DNODE ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_DROP_DNODE, 1, &X); } cmd ::= DROP USER ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_DROP_USER, 1, &X); } cmd ::= DROP ACCOUNT ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_DROP_ACCT, 1, &X); } @@ -136,8 +136,8 @@ cmd ::= DESCRIBE ids(X) cpxName(Y). { /////////////////////////////////THE ALTER STATEMENT//////////////////////////////////////// cmd ::= ALTER USER ids(X) PASS ids(Y). { setAlterUserSQL(pInfo, TSDB_ALTER_USER_PASSWD, &X, &Y, NULL); } cmd ::= ALTER USER ids(X) PRIVILEGE ids(Y). { setAlterUserSQL(pInfo, TSDB_ALTER_USER_PRIVILEGES, &X, NULL, &Y);} -cmd ::= ALTER DNODE IPTOKEN(X) ids(Y). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_DNODE, 2, &X, &Y); } -cmd ::= ALTER DNODE IPTOKEN(X) ids(Y) ids(Z). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_DNODE, 3, &X, &Y, &Z); } +cmd ::= ALTER DNODE ids(X) ids(Y). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_DNODE, 2, &X, &Y); } +cmd ::= ALTER DNODE ids(X) ids(Y) ids(Z). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_DNODE, 3, &X, &Y, &Z); } cmd ::= ALTER LOCAL ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 1, &X); } cmd ::= ALTER LOCAL ids(X) ids(Y). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &X, &Y); } cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SSQLToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);} @@ -162,7 +162,7 @@ ifnotexists(X) ::= . {X.n = 0;} /////////////////////////////////THE CREATE STATEMENT/////////////////////////////////////// //create option for dnode/db/user/account -cmd ::= CREATE DNODE IPTOKEN(X). { setDCLSQLElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &X);} +cmd ::= CREATE DNODE ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &X);} cmd ::= CREATE ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). { setCreateAcctSQL(pInfo, TSDB_SQL_CREATE_ACCT, &X, &Y, &Z);} cmd ::= CREATE DATABASE ifnotexists(Z) ids(X) db_optr(Y). { setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);} diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 7ac0609a55c44de94c350711ff323840d7089375..b148410bfbc14bb681c8217102eeab57b3f9f0c2 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -68,16 +68,18 @@ extern "C" { #define TSDB_FUNC_AVG_RATE 32 #define TSDB_FUNC_AVG_IRATE 33 -#define TSDB_FUNCSTATE_SO 0x1U // single output -#define TSDB_FUNCSTATE_MO 0x2U // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM -#define TSDB_FUNCSTATE_STREAM 0x4U // function avail for stream -#define TSDB_FUNCSTATE_METRIC 0x8U // function avail for metric -#define TSDB_FUNCSTATE_OF 0x10U // outer forward -#define TSDB_FUNCSTATE_NEED_TS 0x20U // timestamp is required during query processing -#define TSDB_FUNCSTATE_SELECTIVITY 0x40U // selectivity functions, can exists along with tag columns +#define TSDB_FUNC_TID_TAG 34 -#define TSDB_BASE_FUNC_SO TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_OF -#define TSDB_BASE_FUNC_MO TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_OF +#define TSDB_FUNCSTATE_SO 0x1u // single output +#define TSDB_FUNCSTATE_MO 0x2u // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM +#define TSDB_FUNCSTATE_STREAM 0x4u // function avail for stream +#define TSDB_FUNCSTATE_STABLE 0x8u // function avail for metric +#define TSDB_FUNCSTATE_OF 0x10u // outer forward +#define TSDB_FUNCSTATE_NEED_TS 0x20u // timestamp is required during query processing +#define TSDB_FUNCSTATE_SELECTIVITY 0x40u // selectivity functions, can exists along with tag columns + +#define TSDB_BASE_FUNC_SO TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_OF +#define TSDB_BASE_FUNC_MO TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_OF #define TSDB_FUNCTIONS_NAME_MAX_LENGTH 16 @@ -108,17 +110,17 @@ enum { SECONDARY_STAGE_MERGE = 0x20u, }; -#define QUERY_IS_STABLE_QUERY(type) (((type)&TSDB_QUERY_TYPE_STABLE_QUERY) != 0) -#define QUERY_IS_JOIN_QUERY(type) (((type)&TSDB_QUERY_TYPE_JOIN_QUERY) != 0) +#define QUERY_IS_STABLE_QUERY(type) (((type)&TSDB_QUERY_TYPE_STABLE_QUERY) != 0) +#define QUERY_IS_JOIN_QUERY(type) (TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_JOIN_QUERY)) #define QUERY_IS_PROJECTION_QUERY(type) (((type)&TSDB_QUERY_TYPE_PROJECTION_QUERY) != 0) -#define QUERY_IS_FREE_RESOURCE(type) (((type)&TSDB_QUERY_TYPE_FREE_RESOURCE) != 0) +#define QUERY_IS_FREE_RESOURCE(type) (((type)&TSDB_QUERY_TYPE_FREE_RESOURCE) != 0) typedef struct SArithmeticSupport { - SArithExprInfo *pArithExpr; - int32_t numOfCols; - SColumnInfo* colList; - int32_t offset; - char** data; + SExprInfo *pArithExpr; + int32_t numOfCols; + SColumnInfo *colList; + int32_t offset; + char** data; } SArithmeticSupport; typedef struct SQLPreAggVal { diff --git a/src/query/src/qast.c b/src/query/src/qast.c index 500a5f1e496c5ac30c3f3d6011098dca0c840be6..e78f36ed31ecc929e75d329d957d5ed24e2635ef 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -37,7 +37,6 @@ * * @date 2018-2-15 * @version 0.2 operation for column filter - * @author liaohj * * @Description parse tag query expression to build ast * ver 0.2, filter the result on first column with high priority to limit the candidate set @@ -468,7 +467,7 @@ void tExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { } typedef struct { - tVariant v; + char* v; int32_t optr; } SEndPoint; @@ -521,21 +520,19 @@ static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) { if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL || optr == TSDB_RELATION_EQUAL || optr == TSDB_RELATION_NOT_EQUAL) { - pCond->start = calloc(1, sizeof(tVariant)); - tVariantAssign(&pCond->start->v, &queryColInfo->q); + pCond->start = calloc(1, sizeof(SEndPoint)); pCond->start->optr = queryColInfo->optr; - + pCond->start->v = queryColInfo->q; } else if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { - pCond->end = calloc(1, sizeof(tVariant)); - tVariantAssign(&pCond->end->v, &queryColInfo->q); + pCond->end = calloc(1, sizeof(SEndPoint)); pCond->end->optr = queryColInfo->optr; - + pCond->end->v = queryColInfo->q; } else if (optr == TSDB_RELATION_IN) { printf("relation is in\n"); - + assert(0); } else if (optr == TSDB_RELATION_LIKE) { printf("relation is like\n"); - + assert(0); } return TSDB_CODE_SUCCESS; @@ -543,18 +540,16 @@ static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) { static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* result) { SSkipListIterator* iter = NULL; - int32_t type = pQueryInfo->q.nType; - - SQueryCond cond = { 0 }; + SQueryCond cond = {0}; setQueryCond(pQueryInfo, &cond); if (cond.start != NULL) { - iter = tSkipListCreateIterFromVal(pSkipList, (char*) &cond.start->v.i64Key, type, TSDB_ORDER_ASC); + iter = tSkipListCreateIterFromVal(pSkipList, (char*) &cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_ASC); } else { - iter = tSkipListCreateIterFromVal(pSkipList, (char*) &cond.end->v.i64Key, type, TSDB_ORDER_DESC); + iter = tSkipListCreateIterFromVal(pSkipList, (char*) &cond.end->v, pSkipList->keyInfo.type, TSDB_ORDER_DESC); } - __compar_fn_t func = getComparFunc(pSkipList->keyInfo.type, type, 0); + __compar_fn_t func = getKeyComparFunc(pSkipList->keyInfo.type); if (cond.start != NULL) { int32_t optr = cond.start->optr; @@ -563,7 +558,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr while(tSkipListIterNext(iter)) { SSkipListNode* pNode = tSkipListIterGet(iter); - int32_t ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &cond.start->v.i64Key); + int32_t ret = func(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v); if (ret == 0) { taosArrayPush(result, SL_GET_NODE_DATA(pNode)); } else { @@ -578,7 +573,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr SSkipListNode* pNode = tSkipListIterGet(iter); if (comp) { - ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &cond.start->v.i64Key); + ret = func(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v); assert(ret >= 0); } @@ -605,7 +600,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr SSkipListNode* pNode = tSkipListIterGet(iter); if (comp) { - ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &cond.end->v.i64Key); + ret = func(SL_GET_NODE_KEY(pSkipList, pNode), cond.end->v); assert(ret <= 0); } @@ -699,21 +694,18 @@ int32_t intersect(SArray *pLeft, SArray *pRight, SArray *pFinalRes) { /* * traverse the result and apply the function to each item to check if the item is qualified or not */ -static UNUSED_FUNC void tSQLListTraverseOnResult(struct tExprNode *pExpr, __result_filter_fn_t fp, SArray *pResult) { -// assert(pExpr->_node.pLeft->nodeType == TSQL_NODE_COL && pExpr->_node.pRight->nodeType == TSQL_NODE_VALUE); -// -// // brutal force scan the result list and check for each item in the list -// int64_t num = pResult->num; -// for (int32_t i = 0, j = 0; i < pResult->num; ++i) { -// if (fp == NULL || (fp(pResult->pRes[i], pExpr->_node.info) == true)) { -// pResult->pRes[j++] = pResult->pRes[i]; -// } else { -// num--; -// } -// } -// -// pResult->num = num; - assert(0); +static void tArrayTraverse(tExprNode *pExpr, __result_filter_fn_t fp, SArray *pResult) { + assert(pExpr->_node.pLeft->nodeType == TSQL_NODE_COL && pExpr->_node.pRight->nodeType == TSQL_NODE_VALUE && fp != NULL); + + // scan the result array list and check for each item in the list + for (int32_t i = 0; i < taosArrayGetSize(pResult); ++i) { + void* item = taosArrayGet(pResult, i); + if (fp(item, pExpr->_node.info)) { + i++; + } else { + taosArrayRemove(pResult, i); + } + } } static bool filterItem(tExprNode *pExpr, const void *pItem, SBinaryFilterSupp *param) { @@ -771,12 +763,7 @@ static void exprTreeTraverseImpl(tExprNode *pExpr, SArray *pResult, SBinaryFilte } -static void tSQLBinaryTraverseOnSkipList( - tExprNode *pExpr, - SArray *pResult, - SSkipList *pSkipList, - SBinaryFilterSupp *param -) { +static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSkipList *pSkipList, SBinaryFilterSupp *param ) { SSkipListIterator* iter = tSkipListCreateIter(pSkipList); while (tSkipListIterNext(iter)) { @@ -797,20 +784,26 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, bool addToResult = false; SSkipListNode *pNode = tSkipListIterGet(iter); - STable* table = *(STable**) SL_GET_NODE_DATA(pNode); + char* pTable = SL_GET_NODE_DATA(pNode); + + //todo refactor: + char* name = (*(STable**) pTable)->name; +// char* name = NULL; +// tsdbGetTableName(tsdb, pTable, &name); + // todo speed up by using hash if (pQueryInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) { if (pQueryInfo->optr == TSDB_RELATION_IN) { - addToResult = pQueryInfo->compare(table->name, pQueryInfo->q.arr); + addToResult = pQueryInfo->compare(name, pQueryInfo->q); } else if(pQueryInfo->optr == TSDB_RELATION_LIKE) { - addToResult = !pQueryInfo->compare(table->name, pQueryInfo->q.pz); + addToResult = !pQueryInfo->compare(name, pQueryInfo->q); } } else { // TODO: other columns } if (addToResult) { - taosArrayPush(result, (void*)&table); + taosArrayPush(result, pTable); } } @@ -834,7 +827,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S param->setupInfoFn(pExpr, param->pExtInfo); if (pSkipList == NULL) { - tSQLListTraverseOnResult(pExpr, param->fp, result); + tArrayTraverse(pExpr, param->fp, result); return; } @@ -920,7 +913,6 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S tExprTreeTraverse(pSecond, NULL, result, param); } - void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*getSourceDataBlock)(void *, const char*, int32_t)) { if (pExprs == NULL) { diff --git a/src/query/src/qextbuffer.c b/src/query/src/qextbuffer.c index 2b403eb92112fc70c8388addd93d0adbbe3d8e5e..adf15d1de0d90e287131bf7b076fa089e4156020 100644 --- a/src/query/src/qextbuffer.c +++ b/src/query/src/qextbuffer.c @@ -409,13 +409,21 @@ static FORCE_INLINE int32_t columnValueAscendingComparator(char *f1, char *f2, i return (first < second) ? -1 : 1; }; case TSDB_DATA_TYPE_BINARY: { - int32_t ret = strncmp(f1, f2, bytes); - if (ret == 0) { - return 0; + int32_t len1 = varDataLen(f1); + int32_t len2 = varDataLen(f2); + + if (len1 != len2) { + return len1 > len2? 1:-1; + } else { + int32_t ret = strncmp(varDataVal(f1), varDataVal(f2), len1); + if (ret == 0) { + return 0; + } + return (ret < 0) ? -1 : 1; } - return (ret < 0) ? -1 : 1; + }; - case TSDB_DATA_TYPE_NCHAR: { + case TSDB_DATA_TYPE_NCHAR: { // todo handle the var string compare int32_t ret = tasoUcs4Compare(f1, f2, bytes); if (ret == 0) { return 0; diff --git a/src/query/src/qparserImpl.c b/src/query/src/qparserImpl.c index 5adb183af3a951b6df708af2a4a3bd618b12ba8d..d8e48de918e9c12ba0a9850dc504a941efc2467c 100644 --- a/src/query/src/qparserImpl.c +++ b/src/query/src/qparserImpl.c @@ -463,7 +463,8 @@ int32_t getTimestampInUsFromStrImpl(int64_t val, char unit, int64_t *result) { void tSQLSetColumnInfo(TAOS_FIELD *pField, SSQLToken *pName, TAOS_FIELD *pType) { int32_t maxLen = sizeof(pField->name) / sizeof(pField->name[0]); - /* truncate the column name */ + + // truncate the column name if (pName->n >= maxLen) { pName->n = maxLen - 1; } @@ -478,7 +479,9 @@ void tSQLSetColumnInfo(TAOS_FIELD *pField, SSQLToken *pName, TAOS_FIELD *pType) void tSQLSetColumnType(TAOS_FIELD *pField, SSQLToken *type) { pField->type = -1; - for (int8_t i = 0; i < sizeof(tDataTypeDesc) / sizeof(tDataTypeDesc[0]); ++i) { + int32_t LENGTH_SIZE_OF_STR = 2; // in case of nchar and binary, there two bytes to keep the length of binary|nchar. + + for (int8_t i = 0; i < tListLen(tDataTypeDesc); ++i) { if ((strncasecmp(type->z, tDataTypeDesc[i].aName, tDataTypeDesc[i].nameLen) == 0) && (type->n == tDataTypeDesc[i].nameLen)) { pField->type = i; @@ -490,10 +493,10 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SSQLToken *type) { * number of bytes in UCS-4 format, which is 4 times larger than the * number of characters */ - pField->bytes = -(int32_t)type->type * TSDB_NCHAR_SIZE; + pField->bytes = -(int32_t)type->type * TSDB_NCHAR_SIZE + LENGTH_SIZE_OF_STR; } else if (i == TSDB_DATA_TYPE_BINARY) { /* for binary, the TOKENTYPE is the length of binary */ - pField->bytes = -(int32_t)type->type; + pField->bytes = -(int32_t) type->type + LENGTH_SIZE_OF_STR; } break; } diff --git a/src/query/src/qresultBuf.c b/src/query/src/qresultBuf.c index 7cdbfbd0ef17bf10ba13722a23e2ef73ad4623f1..bdf700c83f3fa7d69ab83dcb273939b001524cd6 100644 --- a/src/query/src/qresultBuf.c +++ b/src/query/src/qresultBuf.c @@ -7,7 +7,7 @@ #define DEFAULT_INTERN_BUF_SIZE 16384L -int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t size, int32_t rowSize) { +int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t size, int32_t rowSize, void* handle) { SDiskbasedResultBuf* pResBuf = calloc(1, sizeof(SDiskbasedResultBuf)); pResBuf->numOfRowsPerPage = (DEFAULT_INTERN_BUF_SIZE - sizeof(tFilePage)) / rowSize; pResBuf->numOfPages = size; @@ -41,11 +41,13 @@ int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t si pResBuf->pBuf = mmap(NULL, pResBuf->totalBufSize, PROT_READ | PROT_WRITE, MAP_SHARED, pResBuf->fd, 0); if (pResBuf->pBuf == MAP_FAILED) { - qError("QInfo:%p failed to map temp file: %s. %s", pResBuf->path, strerror(errno)); + qError("QInfo:%p failed to map temp file: %s. %s", handle, pResBuf->path, strerror(errno)); return TSDB_CODE_CLI_OUT_OF_MEMORY; // todo change error code } - qTrace("create tmp file for output result, %s, " PRId64 "bytes", pResBuf->path, pResBuf->totalBufSize); + qTrace("QInfo:%p create tmp file for output result, %s, %" PRId64 "bytes", handle, pResBuf->path, + pResBuf->totalBufSize); + *pResultBuf = pResBuf; return TSDB_CODE_SUCCESS; } @@ -189,7 +191,7 @@ SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) } } -void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { +void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle) { if (pResultBuf == NULL) { return; } @@ -198,7 +200,7 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { close(pResultBuf->fd); } - qTrace("disk-based output buffer closed, %" PRId64 " bytes, file:%s", pResultBuf->totalBufSize, pResultBuf->path); + qTrace("QInfo:%p disk-based output buffer closed, %" PRId64 " bytes, file:%s", handle, pResultBuf->totalBufSize, pResultBuf->path); munmap(pResultBuf->pBuf, pResultBuf->totalBufSize); unlink(pResultBuf->path); diff --git a/src/query/src/qtsbuf.c b/src/query/src/qtsbuf.c index 1d5c4f2d9df71d9fc1133e424efeabbc40215043..555ccb7318bed924ededc0fd13cdf1a9c2345997 100644 --- a/src/query/src/qtsbuf.c +++ b/src/query/src/qtsbuf.c @@ -482,7 +482,7 @@ static void tsBufGetBlock(STSBuf* pTSBuf, int32_t vnodeIndex, int32_t blockIndex } STSCursor* pCur = &pTSBuf->cur; - if (pCur->vnodeIndex == vnodeIndex && ((pCur->blockIndex <= blockIndex && pCur->order == TSDB_ORDER_ASC) || + if (pCur->vgroupIndex == vnodeIndex && ((pCur->blockIndex <= blockIndex && pCur->order == TSDB_ORDER_ASC) || (pCur->blockIndex >= blockIndex && pCur->order == TSDB_ORDER_DESC))) { int32_t i = 0; bool decomp = false; @@ -517,7 +517,7 @@ static void tsBufGetBlock(STSBuf* pTSBuf, int32_t vnodeIndex, int32_t blockIndex assert((pTSBuf->tsData.len / TSDB_KEYSIZE == pBlock->numOfElem) && (pTSBuf->tsData.allocSize >= pTSBuf->tsData.len)); - pCur->vnodeIndex = vnodeIndex; + pCur->vgroupIndex = vnodeIndex; pCur->blockIndex = blockIndex; pCur->tsIndex = (pCur->order == TSDB_ORDER_ASC) ? 0 : pBlock->numOfElem - 1; @@ -554,7 +554,7 @@ bool tsBufNextPos(STSBuf* pTSBuf) { STSCursor* pCur = &pTSBuf->cur; // get the first/last position according to traverse order - if (pCur->vnodeIndex == -1) { + if (pCur->vgroupIndex == -1) { if (pCur->order == TSDB_ORDER_ASC) { tsBufGetBlock(pTSBuf, 0, 0); @@ -569,9 +569,9 @@ bool tsBufNextPos(STSBuf* pTSBuf) { assert(pTSBuf->numOfVnodes > 0); int32_t vnodeIndex = pTSBuf->numOfVnodes - 1; - pCur->vnodeIndex = vnodeIndex; + pCur->vgroupIndex = vnodeIndex; - int32_t vnodeId = pTSBuf->pData[pCur->vnodeIndex].info.vnode; + int32_t vnodeId = pTSBuf->pData[pCur->vgroupIndex].info.vnode; STSVnodeBlockInfo* pBlockInfo = tsBufGetVnodeBlockInfo(pTSBuf, vnodeId); int32_t blockIndex = pBlockInfo->numOfBlocks - 1; @@ -594,14 +594,14 @@ bool tsBufNextPos(STSBuf* pTSBuf) { if ((pCur->order == TSDB_ORDER_ASC && pCur->tsIndex >= pTSBuf->block.numOfElem - 1) || (pCur->order == TSDB_ORDER_DESC && pCur->tsIndex <= 0)) { - int32_t vnodeId = pTSBuf->pData[pCur->vnodeIndex].info.vnode; + int32_t vnodeId = pTSBuf->pData[pCur->vgroupIndex].info.vnode; STSVnodeBlockInfo* pBlockInfo = tsBufGetVnodeBlockInfo(pTSBuf, vnodeId); if (pBlockInfo == NULL || (pCur->blockIndex >= pBlockInfo->numOfBlocks - 1 && pCur->order == TSDB_ORDER_ASC) || (pCur->blockIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) { - if ((pCur->vnodeIndex >= pTSBuf->numOfVnodes - 1 && pCur->order == TSDB_ORDER_ASC) || - (pCur->vnodeIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) { - pCur->vnodeIndex = -1; + if ((pCur->vgroupIndex >= pTSBuf->numOfVnodes - 1 && pCur->order == TSDB_ORDER_ASC) || + (pCur->vgroupIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) { + pCur->vgroupIndex = -1; return false; } @@ -610,11 +610,11 @@ bool tsBufNextPos(STSBuf* pTSBuf) { } int32_t blockIndex = pCur->order == TSDB_ORDER_ASC ? 0 : pBlockInfo->numOfBlocks - 1; - tsBufGetBlock(pTSBuf, pCur->vnodeIndex + step, blockIndex); + tsBufGetBlock(pTSBuf, pCur->vgroupIndex + step, blockIndex); break; } else { - tsBufGetBlock(pTSBuf, pCur->vnodeIndex, pCur->blockIndex + step); + tsBufGetBlock(pTSBuf, pCur->vgroupIndex, pCur->blockIndex + step); break; } } else { @@ -631,7 +631,7 @@ void tsBufResetPos(STSBuf* pTSBuf) { return; } - pTSBuf->cur = (STSCursor){.tsIndex = -1, .blockIndex = -1, .vnodeIndex = -1, .order = pTSBuf->cur.order}; + pTSBuf->cur = (STSCursor){.tsIndex = -1, .blockIndex = -1, .vgroupIndex = -1, .order = pTSBuf->cur.order}; } STSElem tsBufGetElem(STSBuf* pTSBuf) { @@ -642,13 +642,13 @@ STSElem tsBufGetElem(STSBuf* pTSBuf) { } STSCursor* pCur = &pTSBuf->cur; - if (pCur != NULL && pCur->vnodeIndex < 0) { + if (pCur != NULL && pCur->vgroupIndex < 0) { return elem1; } STSBlock* pBlock = &pTSBuf->block; - elem1.vnode = pTSBuf->pData[pCur->vnodeIndex].info.vnode; + elem1.vnode = pTSBuf->pData[pCur->vgroupIndex].info.vnode; elem1.ts = *(TSKEY*)(pTSBuf->tsData.rawBuf + pCur->tsIndex * TSDB_KEYSIZE); elem1.tag = pBlock->tag; @@ -804,7 +804,7 @@ STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag) { return elem; } - pCur->vnodeIndex = j; + pCur->vgroupIndex = j; pCur->blockIndex = blockIndex; tsBufGetBlock(pTSBuf, j, blockIndex); @@ -812,7 +812,7 @@ STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag) { } STSCursor tsBufGetCursor(STSBuf* pTSBuf) { - STSCursor c = {.vnodeIndex = -1}; + STSCursor c = {.vgroupIndex = -1}; if (pTSBuf == NULL) { return c; } @@ -825,9 +825,9 @@ void tsBufSetCursor(STSBuf* pTSBuf, STSCursor* pCur) { return; } - // assert(pCur->vnodeIndex != -1 && pCur->tsIndex >= 0 && pCur->blockIndex >= 0); - if (pCur->vnodeIndex != -1) { - tsBufGetBlock(pTSBuf, pCur->vnodeIndex, pCur->blockIndex); + // assert(pCur->vgroupIndex != -1 && pCur->tsIndex >= 0 && pCur->blockIndex >= 0); + if (pCur->vgroupIndex != -1) { + tsBufGetBlock(pTSBuf, pCur->vgroupIndex, pCur->blockIndex); } pTSBuf->cur = *pCur; diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 2143742e24f0929f03d5ca5b962eb883aaf6c7a2..ed0da791dc67cee130ebae13dba203a8cd7aa3b3 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -25,9 +25,9 @@ #include "taosmsg.h" #include "tlosertree.h" #include "tscompression.h" -#include "tsdbMain.h" //todo use TableId instead of STable object #include "ttime.h" #include "tscUtil.h" // todo move the function to common module +#include "tdataformat.h" #define DEFAULT_INTERN_BUF_SIZE 16384L @@ -35,8 +35,6 @@ * check if the primary column is load by default, otherwise, the program will * forced to load primary column explicitly. */ -#define PRIMARY_TSCOL_LOADED(query) ((query)->colList[0].data.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) - #define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0) #define TSDB_COL_IS_TAG(f) (((f)&TSDB_COL_TAG) != 0) #define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP) @@ -58,8 +56,8 @@ typedef struct SPointInterpoSupporter { int32_t numOfCols; - char ** pPrevPoint; - char ** pNextPoint; + SArray* prev; + SArray* next; } SPointInterpoSupporter; typedef enum { @@ -98,12 +96,18 @@ typedef struct { STSCursor cur; } SQueryStatusInfo; +typedef struct SGroupItem { + STableId id; + STableQueryInfo* info; +} SGroupItem; + static void setQueryStatus(SQuery *pQuery, int8_t status); -bool isIntervalQuery(SQuery *pQuery) { return pQuery->intervalTime > 0; } +static bool isIntervalQuery(SQuery *pQuery) { return pQuery->intervalTime > 0; } +// todo move to utility static int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *group); -static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult); +static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult); static void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pResultInfo); static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId); static void getNextTimeWindow(SQuery *pQuery, STimeWindow *pTimeWindow); @@ -114,9 +118,10 @@ static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols); static void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static bool hasMainOutput(SQuery *pQuery); -static void createTableDataInfo(SQInfo *pQInfo); +static void createTableQueryInfo(SQInfo *pQInfo); +static void buildTagQueryResult(SQInfo *pQInfo); -static int32_t setAdditionalInfo(SQInfo *pQInfo, STable *pTable, STableQueryInfo *pTableQueryInfo); +static int32_t setAdditionalInfo(SQInfo *pQInfo, STableId *pTaleId, STableQueryInfo *pTableQueryInfo); static int32_t flushFromResultBuf(SQInfo *pQInfo); bool getNeighborPoints(SQInfo *pQInfo, void *pMeterObj, SPointInterpoSupporter *pPointInterpSupporter) { @@ -926,7 +931,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * TSKEY ts = primaryKeyCol[offset]; STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->sid, &win) != TSDB_CODE_SUCCESS) { + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &win) != TSDB_CODE_SUCCESS) { return; } @@ -947,7 +952,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * } // null data, failed to allocate more memory buffer - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->sid, &nextWin) != TSDB_CODE_SUCCESS) { + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &nextWin) != TSDB_CODE_SUCCESS) { break; } @@ -1178,7 +1183,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS int64_t ts = primaryKeyCol[offset]; STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); - int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->sid, &win); + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &win); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code continue; } @@ -1206,7 +1211,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } // null data, failed to allocate more memory buffer - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->sid, &nextWin) != TSDB_CODE_SUCCESS) { + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &nextWin) != TSDB_CODE_SUCCESS) { break; } @@ -1496,8 +1501,9 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { } SQuery *pQuery = pRuntimeEnv->pQuery; - - qTrace("QInfo:%p teardown runtime env", GET_QINFO_ADDR(pQuery)); + SQInfo* pQInfo = (SQInfo*) GET_QINFO_ADDR(pRuntimeEnv); + + qTrace("QInfo:%p teardown runtime env", pQInfo); cleanupTimeWindowInfo(&pRuntimeEnv->windowResInfo, pQuery->numOfOutput); if (pRuntimeEnv->pCtx != NULL) { @@ -1527,7 +1533,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { tfree(pRuntimeEnv->pInterpoBuf); } - destroyResultBuf(pRuntimeEnv->pResultBuf); + destroyResultBuf(pRuntimeEnv->pResultBuf, pQInfo); tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); @@ -1553,7 +1559,7 @@ static bool isQueryKilled(SQInfo *pQInfo) { static void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_QUERY_CANCELLED; } -bool isFixedOutputQuery(SQuery *pQuery) { +static bool isFixedOutputQuery(SQuery *pQuery) { if (pQuery->intervalTime != 0) { return false; } @@ -1584,7 +1590,7 @@ bool isFixedOutputQuery(SQuery *pQuery) { return false; } -bool isPointInterpoQuery(SQuery *pQuery) { +static bool isPointInterpoQuery(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionID = pQuery->pSelectExpr[i].base.functionId; if (functionID == TSDB_FUNC_INTERP || functionID == TSDB_FUNC_LAST_ROW) { @@ -1596,7 +1602,7 @@ bool isPointInterpoQuery(SQuery *pQuery) { } // TODO REFACTOR:MERGE WITH CLIENT-SIDE FUNCTION -bool isSumAvgRateQuery(SQuery *pQuery) { +static bool isSumAvgRateQuery(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if (functionId == TSDB_FUNC_TS) { @@ -1612,7 +1618,7 @@ bool isSumAvgRateQuery(SQuery *pQuery) { return false; } -bool isFirstLastRowQuery(SQuery *pQuery) { +static bool isFirstLastRowQuery(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionID = pQuery->pSelectExpr[i].base.functionId; if (functionID == TSDB_FUNC_LAST_ROW) { @@ -1623,7 +1629,7 @@ bool isFirstLastRowQuery(SQuery *pQuery) { return false; } -bool notHasQueryTimeRange(SQuery *pQuery) { +static UNUSED_FUNC bool notHasQueryTimeRange(SQuery *pQuery) { return (pQuery->window.skey == 0 && pQuery->window.ekey == INT64_MAX && QUERY_IS_ASC_QUERY(pQuery)) || (pQuery->window.skey == INT64_MAX && pQuery->window.ekey == 0 && (!QUERY_IS_ASC_QUERY(pQuery))); } @@ -1643,6 +1649,18 @@ static bool needReverseScan(SQuery *pQuery) { return false; } + +static bool onlyQueryTags(SQuery* pQuery) { + for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; + if (functionId != TSDB_FUNC_TAGPRJ && functionId != TSDB_FUNC_TID_TAG) { + return false; + } + } + + return true; +} + ///////////////////////////////////////////////////////////////////////////////////////////// void getAlignQueryTimeWindow(SQuery *pQuery, int64_t key, int64_t keyFirst, int64_t keyLast, int64_t *realSkey, @@ -1934,7 +1952,7 @@ static void changeExecuteScanOrder(SQuery *pQuery, bool metricQuery) { } } -static void doSetInterpVal(SQLFunctionCtx *pCtx, TSKEY ts, int16_t type, int32_t index, char *data) { +static UNUSED_FUNC void doSetInterpVal(SQLFunctionCtx *pCtx, TSKEY ts, int16_t type, int32_t index, char *data) { assert(pCtx->param[index].pz == NULL); int32_t len = 0; @@ -1999,6 +2017,7 @@ static void doSetInterpVal(SQLFunctionCtx *pCtx, TSKEY ts, int16_t type, int32_t * @param pInterpoRaw */ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointInterpSupport) { +#if 0 SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; @@ -2008,7 +2027,7 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI } int32_t count = 1; - TSKEY key = *(TSKEY *)pPointInterpSupport->pNextPoint[0]; + TSKEY key = *(TSKEY *)pPointInterpSupport->next[0]; if (key == pQuery->window.skey) { // the queried timestamp has value, return it directly without interpolation @@ -2091,9 +2110,11 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI } } } +#endif } void pointInterpSupporterInit(SQuery *pQuery, SPointInterpoSupporter *pInterpoSupport) { +#if 0 if (isPointInterpoQuery(pQuery)) { pInterpoSupport->pPrevPoint = malloc(pQuery->numOfCols * POINTER_BYTES); pInterpoSupport->pNextPoint = malloc(pQuery->numOfCols * POINTER_BYTES); @@ -2120,9 +2141,11 @@ void pointInterpSupporterInit(SQuery *pQuery, SPointInterpoSupporter *pInterpoSu offset += pQuery->colList[i].bytes; } } +#endif } void pointInterpSupporterDestroy(SPointInterpoSupporter *pPointInterpSupport) { +#if 0 if (pPointInterpSupport->numOfCols <= 0 || pPointInterpSupport->pPrevPoint == NULL) { return; } @@ -2134,6 +2157,7 @@ void pointInterpSupporterDestroy(SPointInterpoSupporter *pPointInterpSupport) { tfree(pPointInterpSupport->pNextPoint); pPointInterpSupport->numOfCols = 0; +#endif } static UNUSED_FUNC void allocMemForInterpo(SQInfo *pQInfo, SQuery *pQuery, void *pMeterObj) { @@ -2238,22 +2262,6 @@ UNUSED_FUNC void vnodeDecMeterRefcnt(SQInfo *pQInfo) { #endif } -UNUSED_FUNC void setTimestampRange(SQueryRuntimeEnv *pRuntimeEnv, int64_t stime, int64_t etime) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionId = pQuery->pSelectExpr[i].base.functionId; - - if (functionId == TSDB_FUNC_SPREAD) { - pRuntimeEnv->pCtx[i].param[1].dKey = stime; - pRuntimeEnv->pCtx[i].param[2].dKey = etime; - - pRuntimeEnv->pCtx[i].param[1].nType = TSDB_DATA_TYPE_DOUBLE; - pRuntimeEnv->pCtx[i].param[2].nType = TSDB_DATA_TYPE_DOUBLE; - } - } -} - static bool needToLoadDataBlock(SQuery *pQuery, SDataStatis *pDataStatis, SQLFunctionCtx *pCtx, int32_t numOfTotalPoints) { if (pDataStatis == NULL) { @@ -2340,7 +2348,7 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBl } if (r == BLK_DATA_NO_NEEDED) { - qTrace("QInfo:%p slot:%d, data block ignored, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_QINFO_ADDR(pRuntimeEnv), + qTrace("QInfo:%p data block ignored, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); } else if (r == BLK_DATA_FILEDS_NEEDED) { if (tsdbRetrieveDataBlockStatisInfo(pRuntimeEnv->pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { @@ -2539,7 +2547,7 @@ static void updatelastkey(SQuery *pQuery, STableQueryInfo *pTableQInfo) { pTable * set tag value in SQLFunctionCtx * e.g.,tag information into input buffer */ -static void doSetTagValueInParam(void *tsdb, STableId id, int32_t tagColId, tVariant *param) { +static void doSetTagValueInParam(void *tsdb, STableId* pTableId, int32_t tagColId, tVariant *param) { tVariantDestroy(param); char * val = NULL; @@ -2547,11 +2555,11 @@ static void doSetTagValueInParam(void *tsdb, STableId id, int32_t tagColId, tVar int16_t type = 0; if (tagColId == TSDB_TBNAME_COLUMN_INDEX) { - tsdbTableGetName(tsdb, id, &val); - bytes = TSDB_TABLE_NAME_LEN; + tsdbGetTableName(tsdb, pTableId, &val); + bytes = strnlen(val, TSDB_TABLE_NAME_LEN); type = TSDB_DATA_TYPE_BINARY; } else { - tsdbGetTableTagVal(tsdb, id, tagColId, &type, &bytes, &val); + tsdbGetTableTagVal(tsdb, pTableId, tagColId, &type, &bytes, &val); } tVariantCreateFromBinary(param, val, bytes, type); @@ -2561,13 +2569,13 @@ static void doSetTagValueInParam(void *tsdb, STableId id, int32_t tagColId, tVar } } -void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, STableId id, void *tsdb) { +void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, STableId* pTableId, void *tsdb) { SQuery *pQuery = pRuntimeEnv->pQuery; SSqlFuncMsg *pFuncMsg = &pQuery->pSelectExpr[0].base; if (pQuery->numOfOutput == 1 && pFuncMsg->functionId == TSDB_FUNC_TS_COMP) { assert(pFuncMsg->numOfParams == 1); - doSetTagValueInParam(tsdb, id, pFuncMsg->arg->argValue.i64, &pRuntimeEnv->pCtx[0].tag); + doSetTagValueInParam(tsdb, pTableId, pFuncMsg->arg->argValue.i64, &pRuntimeEnv->pCtx[0].tag); } else { // set tag value, by which the results are aggregated. for (int32_t idx = 0; idx < pQuery->numOfOutput; ++idx) { @@ -2579,7 +2587,7 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, STableId id, void *tsdb) { } // todo use tag column index to optimize performance - doSetTagValueInParam(tsdb, id, pCol->colId, &pRuntimeEnv->pCtx[idx].tag); + doSetTagValueInParam(tsdb, pTableId, pCol->colId, &pRuntimeEnv->pCtx[idx].tag); } // set the join tag for first column @@ -2744,9 +2752,9 @@ void UNUSED_FUNC displayInterResult(SData **pdata, SQuery *pQuery, int32_t numOf } typedef struct SCompSupporter { - STableDataInfo **pTableDataInfo; - int32_t * position; - SQInfo * pQInfo; + STableQueryInfo **pTableQueryInfo; + int32_t * position; + SQInfo * pQInfo; } SCompSupporter; int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) { @@ -2769,13 +2777,13 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) return -1; } - SWindowResInfo *pWindowResInfo1 = &supporter->pTableDataInfo[left]->pTableQInfo->windowResInfo; + SWindowResInfo *pWindowResInfo1 = &supporter->pTableQueryInfo[left]->windowResInfo; SWindowResult * pWindowRes1 = getWindowResult(pWindowResInfo1, leftPos); char *b1 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes1); TSKEY leftTimestamp = GET_INT64_VAL(b1); - SWindowResInfo *pWindowResInfo2 = &supporter->pTableDataInfo[right]->pTableQInfo->windowResInfo; + SWindowResInfo *pWindowResInfo2 = &supporter->pTableQueryInfo[right]->windowResInfo; SWindowResult * pWindowRes2 = getWindowResult(pWindowResInfo2, rightPos); char *b2 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes2); @@ -2902,16 +2910,16 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { tFilePage **buffer = (tFilePage **)pQuery->sdata; int32_t * posList = calloc(size, sizeof(int32_t)); - STableDataInfo **pTableList = malloc(POINTER_BYTES * size); + STableQueryInfo **pTableList = malloc(POINTER_BYTES * size); // todo opt for the case of one table per group int32_t numOfTables = 0; for (int32_t i = 0; i < size; ++i) { - SPair * p = taosArrayGet(pGroup, i); - STableDataInfo *pInfo = p->sec; + SGroupItem *item = taosArrayGet(pGroup, i); + STableQueryInfo *pInfo = item->info; - SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, pInfo->pTableQInfo->tid); - if (list.size > 0 && pInfo->pTableQInfo->windowResInfo.size > 0) { + SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, pInfo->id.tid); + if (list.size > 0 && pInfo->windowResInfo.size > 0) { pTableList[numOfTables] = pInfo; numOfTables += 1; } @@ -2940,7 +2948,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { while (1) { int32_t pos = pTree->pNode[0].index; - SWindowResInfo *pWindowResInfo = &pTableList[pos]->pTableQInfo->windowResInfo; + SWindowResInfo *pWindowResInfo = &pTableList[pos]->windowResInfo; SWindowResult * pWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]); char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes); @@ -3074,9 +3082,9 @@ void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pRes } } -void setTableDataInfo(STableDataInfo *pTableDataInfo, int32_t tableIndex, int32_t groupId) { - pTableDataInfo->groupIdx = groupId; - pTableDataInfo->tableIndex = tableIndex; +void setTableDataInfo(STableQueryInfo *pTableQueryInfo, int32_t tableIndex, int32_t groupId) { + pTableQueryInfo->groupIdx = groupId; + pTableQueryInfo->tableIndex = tableIndex; } static void doDisableFunctsForSupplementaryScan(SQuery *pQuery, SWindowResInfo *pWindowResInfo, int32_t order) { @@ -3136,7 +3144,7 @@ void disableFuncForReverseScan(SQInfo *pQInfo, int32_t order) { if (isIntervalQuery(pQuery)) { // for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) { - // STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; + // STableQueryInfo *pTableQueryInfo = pQInfo->pTableQueryInfo[i].pTableQInfo; // SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; // // doDisableFunctsForSupplementaryScan(pQuery, pWindowResInfo, order); @@ -3364,7 +3372,7 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } - pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->groupInfo); + pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableIdGroupInfo); setQueryStatus(pQuery, QUERY_NOT_COMPLETED); switchCtxOrder(pRuntimeEnv); @@ -3430,7 +3438,7 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } - pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->groupInfo); + pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableIdGroupInfo); pRuntimeEnv->windowResInfo.curIndex = qstatus.windowIndex; setQueryStatus(pQuery, QUERY_NOT_COMPLETED); @@ -3503,14 +3511,14 @@ static bool hasMainOutput(SQuery *pQuery) { return false; } -STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, int32_t tid, STimeWindow win) { +STableQueryInfo *createTableQueryInfoImpl(SQueryRuntimeEnv *pRuntimeEnv, STableId tableId, STimeWindow win) { STableQueryInfo *pTableQueryInfo = calloc(1, sizeof(STableQueryInfo)); pTableQueryInfo->win = win; pTableQueryInfo->lastKey = win.skey; - pTableQueryInfo->tid = tid; - pTableQueryInfo->cur.vnodeIndex = -1; + pTableQueryInfo->id = tableId; + pTableQueryInfo->cur.vgroupIndex = -1; initWindowResInfo(&pTableQueryInfo->windowResInfo, pRuntimeEnv, 100, 100, TSDB_DATA_TYPE_INT); return pTableQueryInfo; @@ -3544,7 +3552,7 @@ void changeMeterQueryInfoForSuppleQuery(SQuery *pQuery, STableQueryInfo *pTableQ pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; pTableQueryInfo->cur.order = pTableQueryInfo->cur.order ^ 1u; - pTableQueryInfo->cur.vnodeIndex = -1; + pTableQueryInfo->cur.vgroupIndex = -1; } void restoreIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *pTableQueryInfo) { @@ -3562,8 +3570,7 @@ void restoreIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *p * @param pRuntimeEnv * @param pDataBlockInfo */ -void setExecutionContext(SQInfo *pQInfo, STableQueryInfo *pTableQueryInfo, STable *pTable, int32_t groupIdx, - TSKEY nextKey) { +void setExecutionContext(SQInfo *pQInfo, STableQueryInfo *pTableQueryInfo, STableId* pTableId, int32_t groupIdx, TSKEY nextKey) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SWindowResInfo * pWindowResInfo = &pRuntimeEnv->windowResInfo; int32_t GROUPRESULTID = 1; @@ -3588,7 +3595,7 @@ void setExecutionContext(SQInfo *pQInfo, STableQueryInfo *pTableQueryInfo, STabl initCtxOutputBuf(pRuntimeEnv); pTableQueryInfo->lastKey = nextKey; - setAdditionalInfo(pQInfo, pTable, pTableQueryInfo); + setAdditionalInfo(pQInfo, pTableId, pTableQueryInfo); } static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult) { @@ -3616,15 +3623,15 @@ static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult * } } -int32_t setAdditionalInfo(SQInfo *pQInfo, STable *pTable, STableQueryInfo *pTableQueryInfo) { +int32_t setAdditionalInfo(SQInfo *pQInfo, STableId* pTableId, STableQueryInfo *pTableQueryInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; assert(pTableQueryInfo->lastKey >= 0); - setTagVal(pRuntimeEnv, pTable->tableId, pQInfo->tsdb); + setTagVal(pRuntimeEnv, pTableId, pQInfo->tsdb); // both the master and supplement scan needs to set the correct ts comp start position if (pRuntimeEnv->pTSBuf != NULL) { - if (pTableQueryInfo->cur.vnodeIndex == -1) { + if (pTableQueryInfo->cur.vgroupIndex == -1) { pTableQueryInfo->tag = pRuntimeEnv->pCtx[0].tag.i64Key; tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, pTableQueryInfo->tag); @@ -3819,12 +3826,12 @@ void copyFromWindowResToSData(SQInfo *pQInfo, SWindowResult *result) { assert(pQuery->rec.rows <= pQuery->rec.capacity); } -static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv, STableDataInfo *pTableDataInfo) { +static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *pTableQueryInfo) { SQuery *pQuery = pRuntimeEnv->pQuery; // update the number of result for each, only update the number of rows for the corresponding window result. if (pQuery->intervalTime == 0) { - int32_t g = pTableDataInfo->groupIdx; + int32_t g = pTableQueryInfo->groupIdx; assert(pRuntimeEnv->windowResInfo.size > 0); SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, (char *)&g, sizeof(g)); @@ -3834,11 +3841,10 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv, STableDataInf } } -void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, STableDataInfo *pTableDataInfo, +void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *pTableQueryInfo, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, SArray *pDataBlock, __block_search_fn_t searchFn) { SQuery * pQuery = pRuntimeEnv->pQuery; - STableQueryInfo *pTableQueryInfo = pTableDataInfo->pTableQInfo; SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : pDataBlockInfo->rows - 1; @@ -3848,7 +3854,7 @@ void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, STableDataInfo * blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); } - updateWindowResNumOfRes(pRuntimeEnv, pTableDataInfo); + updateWindowResNumOfRes(pRuntimeEnv, pTableQueryInfo); updatelastkey(pQuery, pTableQueryInfo); } @@ -4206,7 +4212,7 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv) { return true; } -int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, bool isSTableQuery) { +int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool isSTableQuery) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; @@ -4225,15 +4231,20 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, bool isSTableQuery) .numOfCols = pQuery->numOfCols, }; - if (!isSTableQuery || isIntervalQuery(pQuery) || isFixedOutputQuery(pQuery)) { - pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->groupInfo); + + // normal query setup the queryhandle here + if (isFirstLastRowQuery(pQuery) && !isSTableQuery) { // in case of last_row query, invoke a different API. + pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableIdGroupInfo); + } else if (!isSTableQuery || isIntervalQuery(pQuery) || isFixedOutputQuery(pQuery)) { + pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableIdGroupInfo); } pQInfo->tsdb = tsdb; + pQInfo->vgId = vgId; pRuntimeEnv->pQuery = pQuery; pRuntimeEnv->pTSBuf = param; - pRuntimeEnv->cur.vnodeIndex = -1; + pRuntimeEnv->cur.vgroupIndex = -1; pRuntimeEnv->stableQuery = isSTableQuery; if (param != NULL) { @@ -4251,7 +4262,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, bool isSTableQuery) if (isSTableQuery) { int32_t rows = getInitialPageNum(pQInfo); - code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize); + code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize, pQInfo); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -4270,7 +4281,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, bool isSTableQuery) } else if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { int32_t rows = getInitialPageNum(pQInfo); - code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize); + code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize, pQInfo); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -4287,36 +4298,36 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, bool isSTableQuery) setQueryStatus(pQuery, QUERY_NOT_COMPLETED); - SPointInterpoSupporter interpInfo = {0}; - pointInterpSupporterInit(pQuery, &interpInfo); +// SPointInterpoSupporter interpInfo = {0}; +// pointInterpSupporterInit(pQuery, &interpInfo); /* - * in case of last_row query without query range, we set the query timestamp to - * pMeterObj->lastKey. Otherwise, keep the initial query time range unchanged. + * in case of last_row query without query range, we set the query timestamp to be + * STable->lastKey. Otherwise, keep the initial query time range unchanged. */ - if (isFirstLastRowQuery(pQuery) && notHasQueryTimeRange(pQuery)) { - if (!normalizeUnBoundLastRowQuery(pQInfo, &interpInfo)) { - sem_post(&pQInfo->dataReady); - pointInterpSupporterDestroy(&interpInfo); - return TSDB_CODE_SUCCESS; - } - } +// if (isFirstLastRowQuery(pQuery)) { +// if (!normalizeUnBoundLastRowQuery(pQInfo, &interpInfo)) { +// sem_post(&pQInfo->dataReady); +// pointInterpSupporterDestroy(&interpInfo); +// return TSDB_CODE_SUCCESS; +// } +// } /* * here we set the value for before and after the specified time into the * parameter for interpolation query */ - pointInterpSupporterSetData(pQInfo, &interpInfo); - pointInterpSupporterDestroy(&interpInfo); +// pointInterpSupporterSetData(pQInfo, &interpInfo); +// pointInterpSupporterDestroy(&interpInfo); - int64_t rs = taosGetIntervalStartTimestamp(pQuery->window.skey, pQuery->intervalTime, pQuery->slidingTimeUnit, - pQuery->precision); - taosInitInterpoInfo(&pRuntimeEnv->interpoInfo, pQuery->order.order, rs, 0, 0); +// int64_t rs = taosGetIntervalStartTimestamp(pQuery->window.skey, pQuery->intervalTime, pQuery->slidingTimeUnit, +// pQuery->precision); +// taosInitInterpoInfo(&pRuntimeEnv->interpoInfo, pQuery->order.order, rs, 0, 0); // allocMemForInterpo(pQInfo, pQuery, pMeterObj); - if (!isPointInterpoQuery(pQuery)) { +// if (!isPointInterpoQuery(pQuery)) { // assert(pQuery->pos >= 0 && pQuery->slot >= 0); - } +// } // the pQuery->window.skey is changed during normalizedFirstQueryRange, so set the newest lastkey value pQuery->lastKey = pQuery->window.skey; @@ -4362,8 +4373,7 @@ static int64_t queryOnDataBlocks(SQInfo *pQInfo) { } SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle); - STableDataInfo *pTableDataInfo = NULL; - STable * pTable = NULL; + STableQueryInfo *pTableQueryInfo = NULL; // todo opt performance using hash table size_t numOfGroup = taosArrayGetSize(pQInfo->groupInfo.pGroupList); @@ -4372,20 +4382,23 @@ static int64_t queryOnDataBlocks(SQInfo *pQInfo) { size_t num = taosArrayGetSize(group); for (int32_t j = 0; j < num; ++j) { - SPair * p = taosArrayGet(group, j); - STableDataInfo *pInfo = p->sec; + SGroupItem *item = taosArrayGet(group, j); + STableQueryInfo *pInfo = item->info; - if (pInfo->pTableQInfo->tid == blockInfo.sid) { - pTableDataInfo = p->sec; - pTable = p->first; + if (pInfo->id.tid == blockInfo.tid) { + assert(pInfo->id.uid == blockInfo.uid); + pTableQueryInfo = item->info; + break; } } + + if (pTableQueryInfo != NULL) { + break; + } } - assert(pTableDataInfo != NULL && pTableDataInfo->pTableQInfo != NULL); - STableQueryInfo *pTableQueryInfo = pTableDataInfo->pTableQInfo; - + assert(pTableQueryInfo != NULL && pTableQueryInfo != NULL); restoreIntervalQueryRange(pRuntimeEnv, pTableQueryInfo); SDataStatis *pStatis = NULL; @@ -4393,10 +4406,10 @@ static int64_t queryOnDataBlocks(SQInfo *pQInfo) { TSKEY nextKey = blockInfo.window.skey; if (!isIntervalQuery(pQuery)) { - setExecutionContext(pQInfo, pTableQueryInfo, pTable, pTableDataInfo->groupIdx, nextKey); + setExecutionContext(pQInfo, pTableQueryInfo, &pTableQueryInfo->id, pTableQueryInfo->groupIdx, nextKey); } else { // interval query setIntervalQueryRange(pTableQueryInfo, pQInfo, nextKey); - int32_t ret = setAdditionalInfo(pQInfo, pTable, pTableQueryInfo); + int32_t ret = setAdditionalInfo(pQInfo, &pTableQueryInfo->id, pTableQueryInfo); if (ret != TSDB_CODE_SUCCESS) { pQInfo->code = ret; @@ -4404,7 +4417,7 @@ static int64_t queryOnDataBlocks(SQInfo *pQInfo) { } } - stableApplyFunctionsOnBlock(pRuntimeEnv, pTableDataInfo, &blockInfo, pStatis, pDataBlock, binarySearchForKey); + stableApplyFunctionsOnBlock(pRuntimeEnv, pTableQueryInfo, &blockInfo, pStatis, pDataBlock, binarySearchForKey); } int64_t et = taosGetTimestampMs(); @@ -4417,27 +4430,24 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { setQueryStatus(pQuery, QUERY_NOT_COMPLETED); SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0); - SPair * p = taosArrayGet(group, index); - - STable * pTable = p->first; - STableDataInfo *pInfo = p->sec; + SGroupItem* item = taosArrayGet(group, index); - setTagVal(pRuntimeEnv, pTable->tableId, pQInfo->tsdb); + setTagVal(pRuntimeEnv, &item->id, pQInfo->tsdb); qTrace("QInfo:%p query on (%d): uid:%" PRIu64 ", tid:%d, qrange:%" PRId64 "-%" PRId64, pQInfo, index, - pTable->tableId.uid, pInfo->pTableQInfo->lastKey, pInfo->pTableQInfo->win.ekey); + item->id.uid, item->id.tid, item->info->lastKey, item->info->win.ekey); STsdbQueryCond cond = { - .twindow = {pInfo->pTableQInfo->lastKey, pInfo->pTableQInfo->win.ekey}, - .order = pQuery->order.order, - .colList = pQuery->colList, + .twindow = {item->info->lastKey, item->info->win.ekey}, + .order = pQuery->order.order, + .colList = pQuery->colList, .numOfCols = pQuery->numOfCols, }; SArray *g1 = taosArrayInit(1, POINTER_BYTES); - SArray *tx = taosArrayInit(1, sizeof(SPair)); + SArray *tx = taosArrayInit(1, sizeof(STableId)); - taosArrayPush(tx, p); + taosArrayPush(tx, &item->info->id); taosArrayPush(g1, &tx); STableGroupInfo gp = {.numOfTables = 1, .pGroupList = g1}; @@ -4452,7 +4462,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { taosArrayDestroy(g1); if (pRuntimeEnv->pTSBuf != NULL) { - if (pRuntimeEnv->cur.vnodeIndex == -1) { + if (pRuntimeEnv->cur.vgroupIndex == -1) { int64_t tag = pRuntimeEnv->pCtx[0].tag.i64Key; STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, tag); @@ -4469,22 +4479,16 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { return true; } -static UNUSED_FUNC int64_t doCheckMetersInGroup(SQInfo *pQInfo, int32_t index, int32_t start) { +static UNUSED_FUNC int64_t doCheckTables(SQInfo *pQInfo, SArray* pTableList) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; - if (!multiTableMultioutputHelper(pQInfo, index)) { + if (!multiTableMultioutputHelper(pQInfo, 0)) { return 0; } SPointInterpoSupporter pointInterpSupporter = {0}; pointInterpSupporterInit(pQuery, &pointInterpSupporter); - assert(0); - - // if (!normalizedFirstQueryRange(dataInDisk, dataInCache, pSupporter, &pointInterpSupporter, NULL)) { - // pointInterpSupporterDestroy(&pointInterpSupporter); - // return 0; - // } /* * here we set the value for before and after the specified time into the @@ -4528,64 +4532,53 @@ static void sequentialTableProcess(SQInfo *pQInfo) { resetCtxOutputBuf(pRuntimeEnv); assert(pQuery->limit.offset == 0 && pQuery->limit.limit != 0); -#if 0 while (pQInfo->groupIndex < numOfGroups) { - SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, pQInfo->groupIndex); - size_t numOfTable = taosArrayGetSize(group); if (isFirstLastRowQuery(pQuery)) { - qTrace("QInfo:%p last_row query on vid:%d, numOfGroups:%d, current group:%d", pQInfo, vid, pTableIdList->numOfSubSet, - pQInfo->groupIndex); - - TSKEY key = -1; - int32_t index = -1; - - // choose the last key for one group - pQInfo->tableIndex = 0; + qTrace("QInfo:%p last_row query on group:%d, total group:%d, current group:%d", pQInfo, pQInfo->groupIndex, + numOfGroups); + + STsdbQueryCond cond = { + .twindow = pQuery->window, + .colList = pQuery->colList, + .order = pQuery->order.order, + .numOfCols = pQuery->numOfCols, + }; + + SArray *g1 = taosArrayInit(1, POINTER_BYTES); + SArray *tx = taosArrayClone(group); + taosArrayPush(g1, &tx); - for (int32_t k = 0; k < numOfTable; ++k, pQInfo->tableIndex++) { - if (isQueryKilled(pQInfo)) { - return; - } + STableGroupInfo gp = {.numOfTables = taosArrayGetSize(tx), .pGroupList = g1}; + + // include only current table + if (pRuntimeEnv->pQueryHandle != NULL) { + tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); + pRuntimeEnv->pQueryHandle = NULL; } - pQuery->window.skey = key; - pQuery->window.ekey = key; + pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(pQInfo->tsdb, &cond, &gp); -// int64_t num = doCheckMetersInGroup(pQInfo, index, start); -// assert(num >= 0); - } else { - qTrace("QInfo:%p interp query on vid:%d, numOfGroups:%d, current group:%d", pQInfo, vid, pTableIdList->numOfSubSet, - pQInfo->groupIndex); + initCtxOutputBuf(pRuntimeEnv); + setTagVal(pRuntimeEnv, (STableId*) taosArrayGet(tx, 0), pQInfo->tsdb); + scanAllDataBlocks(pRuntimeEnv); - for (int32_t k = start; k <= end; ++k) { - if (isQueryKilled(pQInfo)) { - setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); - return; - } - - pQuery->skey = pSupporter->rawSKey; - pQuery->ekey = pSupporter->rawEKey; - - int64_t num = doCheckMetersInGroup(pQInfo, k, start); - if (num == 1) { - break; - } + int64_t numOfRes = getNumOfResult(pRuntimeEnv); + if (numOfRes > 0) { + pQuery->rec.rows += numOfRes; + forwardCtxOutputBuf(pRuntimeEnv, numOfRes); } - } - - pSupporter->groupIndex++; - - // output buffer is full, return to client - if (pQuery->size >= pQuery->pointsToRead) { - break; + + skipResults(pRuntimeEnv); + pQInfo->groupIndex += 1; + + // enable execution for next table, when handling the projection query + enableExecutionForNextTable(pRuntimeEnv); } } -#endif - } else { - createTableDataInfo(pQInfo); + createTableQueryInfo(pQInfo); /* * 1. super table projection query, 2. group-by on normal columns query, 3. ts-comp query @@ -4618,12 +4611,11 @@ static void sequentialTableProcess(SQInfo *pQInfo) { return; } - SPair * p = taosArrayGet(group, pQInfo->tableIndex); - STableDataInfo *pInfo = p->sec; - - TSKEY skey = pInfo->pTableQInfo->lastKey; - if (skey > 0) { - pQuery->window.skey = skey; + SGroupItem *item = taosArrayGet(group, pQInfo->tableIndex); + + STableQueryInfo *pInfo = item->info; + if (pInfo->lastKey > 0) { + pQuery->window.skey = pInfo->lastKey; } if (!multiTableMultioutputHelper(pQInfo, pQInfo->tableIndex)) { @@ -4665,7 +4657,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { * to ensure that, we can reset the query range once query on a meter is completed. */ pQInfo->tableIndex++; - pInfo->pTableQInfo->lastKey = pQuery->lastKey; + pInfo->lastKey = pQuery->lastKey; // if the buffer is full or group by each table, we need to jump out of the loop if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL) /*|| @@ -4731,13 +4723,12 @@ static void sequentialTableProcess(SQInfo *pQInfo) { pQuery->rec.total += pQuery->rec.rows; qTrace( - "QInfo %p, numOfTables:%d, index:%d, numOfGroups:%d, %d points returned, total:%d totalReturn:%d," - " offset:%" PRId64, + "QInfo %p, numOfTables:%d, index:%d, numOfGroups:%d, %d points returned, total:%"PRId64", offset:%" PRId64, pQInfo, pQInfo->groupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, pQuery->rec.rows, pQuery->rec.total, pQuery->limit.offset); } -static void createTableDataInfo(SQInfo *pQInfo) { +static void createTableQueryInfo(SQInfo *pQInfo) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; // todo make sure the table are added the reference count to gauranteed that all involved tables are valid @@ -4749,21 +4740,18 @@ static void createTableDataInfo(SQInfo *pQInfo) { size_t s = taosArrayGetSize(group); for (int32_t j = 0; j < s; ++j) { - SPair *p = (SPair *)taosArrayGet(group, j); + SGroupItem* item = (SGroupItem *)taosArrayGet(group, j); - // STableDataInfo has been created for each table - if (p->sec != NULL) { // todo refactor + // STableQueryInfo has been created for each table + if (item->info != NULL) { return; } - STableDataInfo *pInfo = calloc(1, sizeof(STableDataInfo)); - - setTableDataInfo(pInfo, index, i); - pInfo->pTableQInfo = - createTableQueryInfo(&pQInfo->runtimeEnv, ((STable *)(p->first))->tableId.tid, pQuery->window); - - p->sec = pInfo; - + STableQueryInfo* pInfo = createTableQueryInfoImpl(&pQInfo->runtimeEnv, item->id, pQuery->window); + pInfo->groupIdx = i; + pInfo->tableIndex = index; + + item->info = pInfo; index += 1; } } @@ -4773,7 +4761,7 @@ static void prepareQueryInfoForReverseScan(SQInfo *pQInfo) { // SQuery *pQuery = pQInfo->runtimeEnv.pQuery; // for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) { - // STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; + // STableQueryInfo *pTableQueryInfo = pQInfo->pTableQueryInfo[i].pTableQInfo; // changeMeterQueryInfoForSuppleQuery(pQuery, pTableQueryInfo); // } } @@ -4811,20 +4799,14 @@ static void doCloseAllTimeWindowAfterScan(SQInfo *pQInfo) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; if (isIntervalQuery(pQuery)) { - // for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) { - // STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; - // closeAllTimeWindow(&pTableQueryInfo->windowResInfo); - // } size_t numOfGroup = taosArrayGetSize(pQInfo->groupInfo.pGroupList); for (int32_t i = 0; i < numOfGroup; ++i) { SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, i); size_t num = taosArrayGetSize(group); for (int32_t j = 0; j < num; ++j) { - SPair * p = taosArrayGet(group, j); - STableDataInfo *pInfo = p->sec; - - closeAllTimeWindow(&pInfo->pTableQInfo->windowResInfo); + SGroupItem* item = taosArrayGet(group, j); + closeAllTimeWindow(&item->info->windowResInfo); } } } else { // close results for group result @@ -4865,12 +4847,11 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { pQuery->window.skey, pQuery->window.ekey, pQuery->order.order); // create the query support structures - createTableDataInfo(pQInfo); + createTableQueryInfo(pQInfo); // do check all qualified data blocks int64_t el = queryOnDataBlocks(pQInfo); - qTrace("QInfo:%p forward scan completed, elapsed time: %lldms, reversed scan start, order:%d", pQInfo, el, - pQuery->order.order ^ 1u); + qTrace("QInfo:%p forward scan completed, elapsed time: %lldms, reversed scan start", pQInfo, el); // query error occurred or query is killed, abort current execution if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) { @@ -4900,8 +4881,6 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { } if (isIntervalQuery(pQuery) || isSumAvgRateQuery(pQuery)) { - // assert(pSupporter->groupIndex == 0 && pSupporter->numOfGroupResultPages == 0); - if (mergeIntoGroupResult(pQInfo) == TSDB_CODE_SUCCESS) { copyResToQueryResultBuf(pQInfo, pQuery); @@ -5112,7 +5091,6 @@ static void tableQueryImpl(SQInfo *pQInfo) { pQInfo->pointsInterpo += numOfInterpo; qTrace("QInfo:%p current:%d returned, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total); - sem_post(&pQInfo->dataReady); return; } @@ -5133,7 +5111,6 @@ static void tableQueryImpl(SQInfo *pQInfo) { if (pQuery->rec.rows > 0) { qTrace("QInfo:%p %d rows returned from group results, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total); - sem_post(&pQInfo->dataReady); return; } } @@ -5141,7 +5118,6 @@ static void tableQueryImpl(SQInfo *pQInfo) { qTrace("QInfo:%p query over, %d rows are returned", pQInfo, pQuery->rec.total); // vnodePrintQueryStatistics(pSupporter); - sem_post(&pQInfo->dataReady); return; } @@ -5166,14 +5142,12 @@ static void tableQueryImpl(SQInfo *pQInfo) { /* check if query is killed or not */ if (isQueryKilled(pQInfo)) { qTrace("QInfo:%p query is killed", pQInfo); - } else { - // STableId* pTableId = taosArrayGet(pQInfo->groupInfo, 0); - // qTrace("QInfo:%p uid:%" PRIu64 " tid:%d, query completed, %" PRId64 " rows returned, numOfTotal:%" PRId64 " - // rows", - // pQInfo, pTableId->uid, pTableId->tid, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); + } else {// todo set the table uid and tid in log +// SArray* p = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0); +// SPair* pair = taosArrayGet(p, 0); + qTrace("QInfo:%p query paused, %" PRId64 " rows returned, numOfTotal:%" PRId64 " rows", + pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); } - - sem_post(&pQInfo->dataReady); } static void stableQueryImpl(SQInfo *pQInfo) { @@ -5201,8 +5175,6 @@ static void stableQueryImpl(SQInfo *pQInfo) { pQuery->rec.total); // vnodePrintQueryStatistics(pSupporter); } - - sem_post(&pQInfo->dataReady); } static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pExprMsg, SColumnInfo* pTagCols) { @@ -5235,33 +5207,44 @@ bool validateExprColumnInfo(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pExprMsg, SC return j < pQueryMsg->numOfCols || j < pQueryMsg->numOfTags; } -static int32_t validateQueryMsg(SQueryTableMsg *pQueryMsg) { +static bool validateQueryMsg(SQueryTableMsg *pQueryMsg) { if (pQueryMsg->intervalTime < 0) { - qError("qmsg:%p illegal value of aggTimeInterval %" PRId64 "", pQueryMsg, pQueryMsg->intervalTime); - return -1; - } - - if (pQueryMsg->numOfCols <= 0 || pQueryMsg->numOfCols > TSDB_MAX_COLUMNS) { - qError("qmsg:%p illegal value of numOfCols %d", pQueryMsg, pQueryMsg->numOfCols); - return -1; + qError("qmsg:%p illegal value of interval time %" PRId64 "", pQueryMsg, pQueryMsg->intervalTime); + return false; } if (pQueryMsg->numOfTables <= 0) { qError("qmsg:%p illegal value of numOfTables %d", pQueryMsg, pQueryMsg->numOfTables); - return -1; + return false; } if (pQueryMsg->numOfGroupCols < 0) { qError("qmsg:%p illegal value of numOfGroupbyCols %d", pQueryMsg, pQueryMsg->numOfGroupCols); - return -1; + return false; } if (pQueryMsg->numOfOutput > TSDB_MAX_COLUMNS || pQueryMsg->numOfOutput <= 0) { qError("qmsg:%p illegal value of output columns %d", pQueryMsg, pQueryMsg->numOfOutput); - return -1; + return false; } - return 0; + return true; +} + +static bool validateQuerySourceCols(SQueryTableMsg *pQueryMsg, SSqlFuncMsg** pExprMsg) { + int32_t numOfTotal = pQueryMsg->numOfCols + pQueryMsg->numOfTags; + if (pQueryMsg->numOfCols < 0 || pQueryMsg->numOfTags < 0 || numOfTotal > TSDB_MAX_COLUMNS) { + qError("qmsg:%p illegal value of numOfCols %d numOfTags:%d", pQueryMsg, pQueryMsg->numOfCols, pQueryMsg->numOfTags); + return false; + } else if (numOfTotal == 0) { + for(int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { + if (pExprMsg[i]->functionId != TSDB_FUNC_TAGPRJ) { + return false; + } + } + } + + return true; } static char *createTableIdList(SQueryTableMsg *pQueryMsg, char *pMsg, SArray **pTableIdList) { @@ -5270,11 +5253,11 @@ static char *createTableIdList(SQueryTableMsg *pQueryMsg, char *pMsg, SArray **p *pTableIdList = taosArrayInit(pQueryMsg->numOfTables, sizeof(STableId)); STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; - pTableIdInfo->sid = htonl(pTableIdInfo->sid); + pTableIdInfo->tid = htonl(pTableIdInfo->tid); pTableIdInfo->uid = htobe64(pTableIdInfo->uid); pTableIdInfo->key = htobe64(pTableIdInfo->key); - STableId id = {.uid = pTableIdInfo->uid, .tid = pTableIdInfo->sid}; + STableId id = {.uid = pTableIdInfo->uid, .tid = pTableIdInfo->tid}; taosArrayPush(*pTableIdList, &id); pMsg += sizeof(STableIdInfo); @@ -5282,7 +5265,7 @@ static char *createTableIdList(SQueryTableMsg *pQueryMsg, char *pMsg, SArray **p for (int32_t j = 1; j < pQueryMsg->numOfTables; ++j) { pTableIdInfo = (STableIdInfo *)pMsg; - pTableIdInfo->sid = htonl(pTableIdInfo->sid); + pTableIdInfo->tid = htonl(pTableIdInfo->tid); pTableIdInfo->uid = htobe64(pTableIdInfo->uid); pTableIdInfo->key = htobe64(pTableIdInfo->key); @@ -5328,7 +5311,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->numOfTags = htonl(pQueryMsg->numOfTags); // query msg safety check - if (validateQueryMsg(pQueryMsg) != 0) { + if (!validateQueryMsg(pQueryMsg)) { return TSDB_CODE_INVALID_QUERY_MSG; } @@ -5412,6 +5395,12 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pExprMsg = (SSqlFuncMsg *)pMsg; } + + if (!validateQuerySourceCols(pQueryMsg, *pExpr)) { + tfree(*pExpr); + + return TSDB_CODE_INVALID_QUERY_MSG; + } pMsg = createTableIdList(pQueryMsg, pMsg, pTableIdList); @@ -5476,11 +5465,9 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, strcpy(*tbnameCond, pMsg); pMsg += len; } - - qTrace("qmsg:%p query on %d table(s), qrange:%" PRId64 "-%" PRId64 - ", numOfGroupbyTagCols:%d, ts order:%d, " - "outputCols:%d, numOfCols:%d, interval:%d" PRId64 ", fillType:%d, comptsLen:%d, limit:%" PRId64 - ", offset:%" PRId64, + + qTrace("qmsg:%p query %d tables, qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, order:%d, " + "outputCols:%d, numOfCols:%d, interval:%" PRId64 ", fillType:%d, comptsLen:%d, limit:%" PRId64 ", offset:%" PRId64, pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->window.skey, pQueryMsg->window.ekey, pQueryMsg->numOfGroupCols, pQueryMsg->order, pQueryMsg->numOfOutput, pQueryMsg->numOfCols, pQueryMsg->intervalTime, pQueryMsg->interpoType, pQueryMsg->tsLen, pQueryMsg->limit, pQueryMsg->offset); @@ -5488,10 +5475,17 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, return 0; } -static int32_t buildAirthmeticExprFromMsg(SArithExprInfo *pArithExprInfo, SQueryTableMsg *pQueryMsg) { +static int32_t buildAirthmeticExprFromMsg(SExprInfo *pArithExprInfo, SQueryTableMsg *pQueryMsg) { qTrace("qmsg:%p create arithmetic expr from binary string", pQueryMsg, pArithExprInfo->base.arg[0].argValue.pz); - - tExprNode* pExprNode = exprTreeFromBinary(pArithExprInfo->base.arg[0].argValue.pz, pArithExprInfo->base.arg[0].argBytes); + + tExprNode* pExprNode = NULL; + TRY(32) { + pExprNode = exprTreeFromBinary(pArithExprInfo->base.arg[0].argValue.pz, pArithExprInfo->base.arg[0].argBytes); + } CATCH( code ) { + CLEANUP_EXECUTE(); + return code; + } END_TRY + if (pExprNode == NULL) { qError("qmsg:%p failed to create arithmetic expression string from:%s", pQueryMsg, pArithExprInfo->base.arg[0].argValue.pz); return TSDB_CODE_APP_ERROR; @@ -5501,12 +5495,12 @@ static int32_t buildAirthmeticExprFromMsg(SArithExprInfo *pArithExprInfo, SQuery return TSDB_CODE_SUCCESS; } -static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SArithExprInfo **pSqlFuncExpr, +static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo **pSqlFuncExpr, SSqlFuncMsg **pExprMsg, SColumnInfo* pTagCols) { *pSqlFuncExpr = NULL; int32_t code = TSDB_CODE_SUCCESS; - SArithExprInfo *pExprs = (SArithExprInfo *)calloc(1, sizeof(SArithExprInfo) * pQueryMsg->numOfOutput); + SExprInfo *pExprs = (SExprInfo *)calloc(1, sizeof(SExprInfo) * pQueryMsg->numOfOutput); if (pExprs == NULL) { return TSDB_CODE_SERV_OUT_OF_MEMORY; } @@ -5537,7 +5531,7 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SArithExp bytes = TSDB_TABLE_NAME_LEN; } else{ int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); - assert(j < pQueryMsg->numOfCols); + assert(j < pQueryMsg->numOfCols || j < pQueryMsg->numOfTags); SColumnInfo* pCol = (TSDB_COL_IS_TAG(pExprs[i].base.colInfo.flag))? &pTagCols[j]:&pQueryMsg->colList[j]; type = pCol->type; @@ -5720,7 +5714,7 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { } } -static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SArithExprInfo *pExprs, +static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, STableGroupInfo *groupInfo, SColumnInfo* pTagCols) { SQInfo *pQInfo = (SQInfo *)calloc(1, sizeof(SQInfo)); if (pQInfo == NULL) { @@ -5734,7 +5728,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou int16_t numOfOutput = pQueryMsg->numOfOutput; pQuery->numOfCols = numOfCols; - pQuery->numOfOutput = numOfOutput; + pQuery->numOfOutput = numOfOutput; pQuery->limit.limit = pQueryMsg->limit; pQuery->limit.offset = pQueryMsg->offset; pQuery->order.order = pQueryMsg->order; @@ -5808,7 +5802,26 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou // to make sure third party won't overwrite this structure pQInfo->signature = pQInfo; - pQInfo->groupInfo = *groupInfo; + + pQInfo->tableIdGroupInfo = *groupInfo; + size_t numOfGroups = taosArrayGetSize(groupInfo->pGroupList); + + pQInfo->groupInfo.pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES); + pQInfo->groupInfo.numOfTables = groupInfo->numOfTables; + + for(int32_t i = 0; i < numOfGroups; ++i) { + SArray* pa = taosArrayGetP(groupInfo->pGroupList, i); + size_t s = taosArrayGetSize(pa); + + SArray* p1 = taosArrayInit(s, sizeof(SGroupItem)); + + for(int32_t j = 0; j < s; ++j) { + SGroupItem item = { .id = *(STableId*) taosArrayGet(pa, j), .info = NULL, }; + taosArrayPush(p1, &item); + } + + taosArrayPush(pQInfo->groupInfo.pGroupList, &p1); + } pQuery->pos = -1; @@ -5860,8 +5873,9 @@ static bool isValidQInfo(void *param) { return (sig == (uint64_t)pQInfo); } -static void freeQInfo(SQInfo *pQInfo); -static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, SQInfo *pQInfo, bool isSTable) { +static void freeQInfo(SQInfo *pQInfo); + +static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *pQInfo, bool isSTable) { int32_t code = TSDB_CODE_SUCCESS; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; @@ -5879,14 +5893,14 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, SQInfo *pQInfo, (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.ekey > pQuery->window.skey))) { qTrace("QInfo:%p no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo, pQuery->window.skey, pQuery->window.ekey, pQuery->order.order); + setQueryStatus(pQuery, QUERY_COMPLETED); sem_post(&pQInfo->dataReady); - setQueryStatus(pQuery, QUERY_COMPLETED); return TSDB_CODE_SUCCESS; } // filter the qualified - if ((code = doInitQInfo(pQInfo, pTSBuf, tsdb, isSTable)) != TSDB_CODE_SUCCESS) { + if ((code = doInitQInfo(pQInfo, pTSBuf, tsdb, vgId, isSTable)) != TSDB_CODE_SUCCESS) { goto _error; } @@ -5925,12 +5939,11 @@ static void freeQInfo(SQInfo *pQInfo) { if (pQuery->pSelectExpr != NULL) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { -// SExprInfo *pBinExprInfo = &pQuery->pSelectExpr[i].arithExprInfo; + SExprInfo* pExprInfo = &pQuery->pSelectExpr[i]; -// if (pBinExprInfo->numOfCols > 0) { -// tfree(pBinExprInfo->pReqColumns); -// tExprTreeDestroy(&pBinExprInfo->pBinExpr, NULL); -// } + if (pExprInfo->pExpr != NULL) { + tExprTreeDestroy(&pExprInfo->pExpr, NULL); + } } tfree(pQuery->pSelectExpr); @@ -5947,15 +5960,24 @@ static void freeQInfo(SQInfo *pQInfo) { size_t num = taosArrayGetSize(p); for(int32_t j = 0; j < num; ++j) { - SPair* pair = taosArrayGet(p, j); - if (pair->sec != NULL) { - destroyTableQueryInfo(((STableDataInfo*)pair->sec)->pTableQInfo, pQuery->numOfOutput); - tfree(pair->sec); + SGroupItem* item = taosArrayGet(p, j); + if (item->info != NULL) { + destroyTableQueryInfo(item->info, pQuery->numOfOutput); } } + + taosArrayDestroy(p); + } + + taosArrayDestroy(pQInfo->groupInfo.pGroupList); + + for(int32_t i = 0; i < numOfGroups; ++i) { + SArray* p = taosArrayGetP(pQInfo->tableIdGroupInfo.pGroupList, i); taosArrayDestroy(p); } + taosArrayDestroy(pQInfo->tableIdGroupInfo.pGroupList); + if (pQuery->pGroupbyExpr != NULL) { taosArrayDestroy(pQuery->pGroupbyExpr->columnInfo); tfree(pQuery->pGroupbyExpr); @@ -5966,7 +5988,6 @@ static void freeQInfo(SQInfo *pQInfo) { tfree(pQuery->colList); tfree(pQuery->sdata); - taosArrayDestroy(pQInfo->groupInfo.pGroupList); tfree(pQuery); qTrace("QInfo:%p QInfo is freed", pQInfo); @@ -6017,9 +6038,15 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { unlink(pQuery->sdata[0]->data); } else { + // todo return the error code to client qError("QInfo:%p failed to open tmp file to send ts-comp data to client, path:%s, reason:%s", pQInfo, pQuery->sdata[0]->data, strerror(errno)); } + + // all data returned, set query over + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { + setQueryStatus(pQuery, QUERY_OVER); + } } else { doCopyQueryResultToMsg(pQInfo, pQuery->rec.rows, data); } @@ -6032,7 +6059,7 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { // todo if interpolation exists, the result may be dump to client by several rounds } -int32_t qCreateQueryInfo(void *tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) { +int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) { assert(pQueryMsg != NULL); int32_t code = TSDB_CODE_SUCCESS; @@ -6060,7 +6087,7 @@ int32_t qCreateQueryInfo(void *tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) goto _query_over; } - SArithExprInfo *pExprs = NULL; + SExprInfo *pExprs = NULL; if ((code = createSqlFunctionExprFromMsg(pQueryMsg, &pExprs, pExprMsg, pTagColumnInfo)) != TSDB_CODE_SUCCESS) { goto _query_over; } @@ -6070,14 +6097,19 @@ int32_t qCreateQueryInfo(void *tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) goto _query_over; } - bool isSTableQuery = false; + bool isSTableQuery = false; STableGroupInfo groupInfo = {0}; - - if ((pQueryMsg->queryType & TSDB_QUERY_TYPE_STABLE_QUERY) != 0) { + + if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_TABLE_QUERY)) { + isSTableQuery = TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY); + + STableId *id = taosArrayGet(pTableIdList, 0); + if ((code = tsdbGetOneTableGroup(tsdb, id->uid, &groupInfo)) != TSDB_CODE_SUCCESS) { + goto _query_over; + } + } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_STABLE_QUERY)) { isSTableQuery = true; - STableId *id = taosArrayGet(pTableIdList, 0); - id->uid = -1; // todo fix me // group by normal column, do not pass the group by condition to tsdb to group table into different group int32_t numOfGroupByCols = pQueryMsg->numOfGroupCols; @@ -6086,19 +6118,14 @@ int32_t qCreateQueryInfo(void *tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) } // todo handle the error - /*int32_t ret =*/tsdbQueryByTagsCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, pQueryMsg->tagNameRelType, tbnameCond, &groupInfo, pGroupColIndex, + /*int32_t ret =*/tsdbQuerySTableByTagCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, pQueryMsg->tagNameRelType, tbnameCond, &groupInfo, pGroupColIndex, numOfGroupByCols); if (groupInfo.numOfTables == 0) { // no qualified tables no need to do query code = TSDB_CODE_SUCCESS; goto _query_over; } } else { - assert(taosArrayGetSize(pTableIdList) == 1); - - STableId *id = taosArrayGet(pTableIdList, 0); - if ((code = tsdbGetOneTableGroup(tsdb, id->uid, &groupInfo)) != TSDB_CODE_SUCCESS) { - goto _query_over; - } + assert(0); } (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, &groupInfo, pTagColumnInfo); @@ -6106,7 +6133,7 @@ int32_t qCreateQueryInfo(void *tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) code = TSDB_CODE_SERV_OUT_OF_MEMORY; } - code = initQInfo(pQueryMsg, tsdb, *pQInfo, isSTableQuery); + code = initQInfo(pQueryMsg, tsdb, vgId, *pQInfo, isSTableQuery); _query_over: tfree(tagCond); @@ -6115,7 +6142,7 @@ _query_over: // if failed to add ref for all meters in this query, abort current query // atomic_fetch_add_32(&vnodeSelectReqNum, 1); - return TSDB_CODE_SUCCESS; + return code; } void qDestroyQueryInfo(qinfo_t pQInfo) { @@ -6137,13 +6164,16 @@ void qTableQuery(qinfo_t qinfo) { } qTrace("QInfo:%p query task is launched", pQInfo); - - if (pQInfo->runtimeEnv.stableQuery) { + + if (onlyQueryTags(pQInfo->runtimeEnv.pQuery)) { + buildTagQueryResult(pQInfo); // todo support the limit/offset + } else if (pQInfo->runtimeEnv.stableQuery) { stableQueryImpl(pQInfo); } else { tableQueryImpl(pQInfo); } - + + sem_post(&pQInfo->dataReady); // vnodeDecRefCount(pQInfo); } @@ -6229,3 +6259,80 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co // pObj->qhandle = NULL; // } } + +static void buildTagQueryResult(SQInfo* pQInfo) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + + size_t num = taosArrayGetSize(pQInfo->groupInfo.pGroupList); + assert(num == 1); // only one group + + SArray* pa = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0); + num = taosArrayGetSize(pa); + + assert(num == pQInfo->groupInfo.numOfTables); + int16_t type, bytes; + + int32_t functionId = pQuery->pSelectExpr[0].base.functionId; + if (functionId == TSDB_FUNC_TID_TAG) { // return the tags & table Id + assert(pQuery->numOfOutput == 1); + SExprInfo* pExprInfo = &pQuery->pSelectExpr[0]; + + int32_t rsize = pExprInfo->bytes; + char* data = NULL; + + for(int32_t i = 0; i < num; ++i) { + SGroupItem* item = taosArrayGet(pa, i); + + char* output = pQuery->sdata[0]->data + i * rsize; + *(int64_t*) output = item->id.uid; // memory align problem + output += sizeof(item->id.uid); + + *(int32_t*) output = item->id.tid; + output += sizeof(item->id.tid); + + *(int32_t*) output = pQInfo->vgId; + output += sizeof(pQInfo->vgId); + + tsdbGetTableTagVal(pQInfo->tsdb, &item->id, pExprInfo->base.colInfo.colId, &type, &bytes, &data); + memcpy(output, data, bytes); + } + + qTrace("QInfo:%p create (tableId, tag) info completed, rows:%d", pQInfo, num); + } else { // return only the tags|table name etc. + for(int32_t i = 0; i < num; ++i) { + SExprInfo* pExprInfo = pQuery->pSelectExpr; + SGroupItem* item = taosArrayGet(pa, i); + + char* data = NULL; + for(int32_t j = 0; j < pQuery->numOfOutput; ++j) { + // todo check the return value, refactor codes + if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { + tsdbGetTableName(pQInfo->tsdb, &item->id, &data); + + char* dst = pQuery->sdata[j]->data + i * (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE); + STR_WITH_MAXSIZE_TO_VARSTR(dst, data, TSDB_TABLE_NAME_LEN); + tfree(data); + + } else {// todo refactor, return the true length of binary|nchar data + tsdbGetTableTagVal(pQInfo->tsdb, &item->id, pExprInfo[j].base.colInfo.colId, &type, &bytes, &data); + assert(bytes == pExprInfo[j].bytes && type == pExprInfo[j].type); + + char* dst = pQuery->sdata[j]->data + i * bytes; + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + memcpy(dst, data, varDataTLen(data)); + } else { + memcpy(dst, data, bytes); + } + } + + } + } + + qTrace("QInfo:%p create tag values results completed, rows:%d", pQInfo, num); + } + + pQuery->rec.rows = num; + setQueryStatus(pQuery, QUERY_COMPLETED); +} + diff --git a/src/query/src/tvariant.c b/src/query/src/tvariant.c index ab5e30f6c40609068496098b4846e7783c09faf3..0c4a2b67578c8f59643941d5f5ac3caaceebce54 100644 --- a/src/query/src/tvariant.c +++ b/src/query/src/tvariant.c @@ -408,7 +408,7 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) { if (*pDest == pVariant->pz) { wchar_t *pWStr = calloc(1, (nLen + 1) * TSDB_NCHAR_SIZE); - taosMbsToUcs4(pDst, nLen, (char *)pWStr, (nLen + 1) * TSDB_NCHAR_SIZE); + taosMbsToUcs4(pDst, nLen, (char *)pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL); // free the binary buffer in the first place if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { @@ -424,7 +424,7 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) { pVariant->wpz = (wchar_t *)tmp; } else { - taosMbsToUcs4(pDst, nLen, *pDest, (nLen + 1) * TSDB_NCHAR_SIZE); + taosMbsToUcs4(pDst, nLen, *pDest, (nLen + 1) * TSDB_NCHAR_SIZE, NULL); } return 0; diff --git a/src/query/tests/resultBufferTest.cpp b/src/query/tests/resultBufferTest.cpp index fca9ac8f9d82911975dc9e917b48b93be26142de..9270c6e45865216260894709e42a439696d99684 100644 --- a/src/query/tests/resultBufferTest.cpp +++ b/src/query/tests/resultBufferTest.cpp @@ -10,7 +10,7 @@ namespace { // simple test void simpleTest() { SDiskbasedResultBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1000, 64); + int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1000, 64, NULL); int32_t pageId = 0; int32_t groupId = 0; @@ -26,7 +26,7 @@ void simpleTest() { ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1); - destroyResultBuf(pResultBuf); + destroyResultBuf(pResultBuf, NULL); } } // namespace diff --git a/src/rpc/src/rpcCache.c b/src/rpc/src/rpcCache.c index ab9e679acf57cdbadb3a0c71692bfd9c030cee1a..edbb9b3e12be6dc24acfbb60fff1f91fe8e1c898 100644 --- a/src/rpc/src/rpcCache.c +++ b/src/rpc/src/rpcCache.c @@ -146,7 +146,6 @@ void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, in rpcUnlockCache(pCache->lockedBy+hash); pCache->total++; - tTrace("%p %s:%hu:%d:%d:%p added into cache, connections:%d", data, fqdn, port, connType, hash, pNode, pCache->count[hash]); return; @@ -202,6 +201,8 @@ void *rpcGetConnFromCache(void *handle, char *fqdn, uint16_t port, int8_t connTy if (pData) { tTrace("%p %s:%hu:%d:%d:%p retrieved from cache, connections:%d", pData, fqdn, port, connType, hash, pNode, pCache->count[hash]); + } else { + tTrace("%s:%hu:%d:%d failed to retrieve conn from cache, connections:%d", fqdn, port, connType, hash, pCache->count[hash]); } return pData; diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 36d74189f5045c7a7d8321ca5c37844e5dbc7aa2..a6d37d6978a68269d0273321cea0bc2860d803e4 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -32,6 +32,8 @@ #include "rpcCache.h" #include "rpcTcp.h" #include "rpcHead.h" +#include "shash.h" + #define RPC_MSG_OVERHEAD (sizeof(SRpcReqContext) + sizeof(SRpcHead) + sizeof(SRpcDigest)) #define rpcHeadFromCont(cont) ((SRpcHead *) (cont - sizeof(SRpcHead))) @@ -260,7 +262,9 @@ void *rpcOpen(const SRpcInit *pInit) { } if (pRpc->connType == TAOS_CONN_SERVER) { - pRpc->hash = taosHashInit(pRpc->sessions, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true); + pRpc->hash = taosInitStrHash(pRpc->sessions, sizeof(pRpc), taosHashString); + +// pRpc->hash = taosHashInit(pRpc->sessions, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true); if (pRpc->hash == NULL) { tError("%s failed to init string hash", pRpc->label); rpcClose(pRpc); @@ -294,7 +298,8 @@ void rpcClose(void *param) { (*taosCleanUpConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle); (*taosCleanUpConn[pRpc->connType])(pRpc->udphandle); - taosHashCleanup(pRpc->hash); +// taosHashCleanup(pRpc->hash); + taosCleanUpStrHash(pRpc->hash); taosTmrCleanUp(pRpc->tmrCtrl); taosIdPoolCleanUp(pRpc->idPool); rpcCloseConnCache(pRpc->pCache); @@ -543,9 +548,10 @@ static void rpcCloseConn(void *thandle) { if ( pRpc->connType == TAOS_CONN_SERVER) { char hashstr[40] = {0}; - size_t size = sprintf(hashstr, "%x:%x:%x:%d", pConn->peerIp, pConn->linkUid, pConn->peerId, pConn->connType); - taosHashRemove(pRpc->hash, hashstr, size); - + /*size_t size = */sprintf(hashstr, "%x:%x:%x:%d", pConn->peerIp, pConn->linkUid, pConn->peerId, pConn->connType); +// taosHashRemove(pRpc->hash, hashstr, size); + taosDeleteStrHash(pRpc->hash, hashstr); + rpcFreeMsg(pConn->pRspMsg); // it may have a response msg saved, but not request msg pConn->pRspMsg = NULL; pConn->inType = 0; @@ -593,10 +599,12 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { char hashstr[40] = {0}; SRpcHead *pHead = (SRpcHead *)pRecv->msg; - size_t size = sprintf(hashstr, "%x:%x:%x:%d", pRecv->ip, pHead->linkUid, pHead->sourceId, pRecv->connType); + /*size_t size = */sprintf(hashstr, "%x:%x:%x:%d", pRecv->ip, pHead->linkUid, pHead->sourceId, pRecv->connType); // check if it is already allocated - SRpcConn **ppConn = (SRpcConn **)(taosHashGet(pRpc->hash, hashstr, size)); + SRpcConn **ppConn = (SRpcConn **)(taosGetStrHashData(pRpc->hash, hashstr)); + +// SRpcConn **ppConn = (SRpcConn **)(taosHashGet(pRpc->hash, hashstr, size)); if (ppConn) pConn = *ppConn; if (pConn) return pConn; @@ -629,8 +637,10 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { pRpc->index = (pRpc->index+1) % pRpc->numOfThreads; pConn->localPort = (pRpc->localPort + pRpc->index); } - - taosHashPut(pRpc->hash, hashstr, size, (char *)&pConn, POINTER_BYTES); + + taosAddStrHash(pRpc->hash, hashstr, (char *)&pConn); + +// taosHashPut(pRpc->hash, hashstr, size, (char *)&pConn, POINTER_BYTES); tTrace("%s %p, rpc connection is allocated, sid:%d id:%s port:%u", pRpc->label, pConn, sid, pConn->user, pConn->localPort); diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 509de6a921018f7d09eec6899e7b13092d999890..ec4fb2a33f91db36bbae2f55aac8312fc7874610 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -225,35 +225,47 @@ STSchema * tsdbGetTableTagSchema(STsdbMeta *pMeta, STable *pTable) { } } -int32_t tsdbGetTableTagVal(TsdbRepoT* repo, STableId id, int32_t colId, int16_t* type, int16_t* bytes, char** val) { +int32_t tsdbGetTableTagVal(TsdbRepoT* repo, STableId* id, int32_t colId, int16_t* type, int16_t* bytes, char** val) { STsdbMeta* pMeta = tsdbGetMeta(repo); - STable* pTable = tsdbGetTableByUid(pMeta, id.uid); + STable* pTable = tsdbGetTableByUid(pMeta, id->uid); STSchema* pSchema = tsdbGetTableTagSchema(pMeta, pTable); STColumn* pCol = NULL; + int32_t offset = 0; for(int32_t col = 0; col < schemaNCols(pSchema); ++col) { STColumn* p = schemaColAt(pSchema, col); if (p->colId == colId) { pCol = p; + break; + } + + if (p->type == TSDB_DATA_TYPE_BINARY || p->type == TSDB_DATA_TYPE_NCHAR) { + offset += sizeof(int32_t); + } else { + offset += p->bytes; } } + if (pCol == NULL) { + return -1; // No matched tags. Maybe the modification of tags has not been done yet. + } + assert(pCol != NULL); SDataRow row = (SDataRow)pTable->tagVal; - char* d = dataRowTuple(row); + char* d = tdGetRowDataOfCol(row, pCol->type, TD_DATA_ROW_HEAD_SIZE + offset); *val = d; *type = pCol->type; *bytes = pCol->bytes; - return 0; + return TSDB_CODE_SUCCESS; } -int32_t tsdbTableGetName(TsdbRepoT *repo, STableId id, char** name) { +int32_t tsdbGetTableName(TsdbRepoT *repo, STableId* id, char** name) { STsdbMeta* pMeta = tsdbGetMeta(repo); - STable* pTable = tsdbGetTableByUid(pMeta, id.uid); + STable* pTable = tsdbGetTableByUid(pMeta, id->uid); *name = strndup(pTable->name, TSDB_TABLE_NAME_LEN); if (*name == NULL) { diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 46480b2b9d06070a06c878c71f17c9fff7edd3db..6d571bd035149a2090a25e607f46c8a867b751ca 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -26,7 +26,6 @@ #include "tsdbMain.h" #define EXTRA_BYTES 2 -#define PRIMARY_TSCOL_REQUIRED(c) (((SColumnInfoData*)taosArrayGet(c, 0))->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) #define ASCENDING_ORDER_TRAVERSE(o) (o == TSDB_ORDER_ASC) #define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns))) @@ -35,6 +34,12 @@ enum { QUERY_RANGE_GREATER_EQUAL = 1, }; +enum { + TSDB_QUERY_TYPE_ALL = 1, + TSDB_QUERY_TYPE_LAST = 2, + TSDB_QUERY_TYPE_EXTERNAL = 3, +}; + typedef struct SField { // todo need the definition } SField; @@ -54,7 +59,7 @@ typedef struct SDataBlockLoadInfo { } SDataBlockLoadInfo; typedef struct SLoadCompBlockInfo { - int32_t sid; /* meter sid */ + int32_t sid; /* table sid */ int32_t fileId; int32_t fileListIndex; } SLoadCompBlockInfo; @@ -63,16 +68,12 @@ typedef struct STableCheckInfo { STableId tableId; TSKEY lastKey; STable* pTableObj; - int64_t offsetInHeaderFile; int32_t start; - bool checkFirstFileBlock; - SCompInfo* pCompInfo; int32_t compSize; - int32_t numOfBlocks; // number of qualified data blocks not the original blocks - - SDataCols* pDataCols; + SDataCols* pDataCols; + SSkipListIterator* iter; } STableCheckInfo; @@ -92,7 +93,7 @@ typedef struct SBlockOrderSupporter { int32_t numOfTables; STableBlockInfo** pDataBlockInfo; int32_t* blockIndexArray; - int32_t* numOfBlocksPerMeter; + int32_t* numOfBlocksPerTable; } SBlockOrderSupporter; typedef struct STsdbQueryHandle { @@ -107,22 +108,23 @@ typedef struct STsdbQueryHandle { SCompBlock* pBlock; int32_t numOfBlocks; SField** pFields; - SArray* pColumns; // column list, SColumnInfoData array list + SArray* pColumns; // column list, SColumnInfoData array list bool locateStart; int32_t realNumOfRows; - SArray* pTableCheckInfo; + SArray* pTableCheckInfo; //SArray int32_t activeIndex; bool checkFiles; // check file stage - void* qinfo; // query info handle, for debug purpose - + void* qinfo; // query info handle, for debug purpose + int32_t type; // query type: retrieve all data blocks, 2. retrieve only last row, 3. retrieve direct prev|next rows STableBlockInfo* pDataBlockInfo; SFileGroup* pFileGroup; SFileGroupIter fileIter; - SCompIdx* compIndex; - SRWHelper rhelper; + SRWHelper rhelper; } STsdbQueryHandle; +static void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle); + static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { pBlockLoadInfo->slot = -1; pBlockLoadInfo->sid = -1; @@ -143,7 +145,7 @@ TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STable pQueryHandle->order = pCond->order; pQueryHandle->window = pCond->twindow; pQueryHandle->pTsdb = tsdb; - pQueryHandle->compIndex = calloc(10000, sizeof(SCompIdx)); + pQueryHandle->type = TSDB_QUERY_TYPE_ALL; tsdbInitReadHelper(&pQueryHandle->rhelper, (STsdbRepo*) tsdb); pQueryHandle->cur.fid = -1; @@ -160,15 +162,15 @@ TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STable assert(gsize > 0); for (int32_t j = 0; j < gsize; ++j) { - SPair* d = (SPair*) taosArrayGet(group, j); - assert(d->first != NULL); + STableId* id = (STableId*) taosArrayGet(group, j); STableCheckInfo info = { .lastKey = pQueryHandle->window.skey, - .tableId = ((STable*) d->first)->tableId, - .pTableObj = d->first, + .tableId = *id, + .pTableObj = tsdbGetTableByUid(tsdbGetMeta(tsdb), id->uid), }; + assert(info.pTableObj != NULL && info.pTableObj->tableId.tid == id->tid); taosArrayPush(pQueryHandle->pTableCheckInfo, &info); } } @@ -198,7 +200,27 @@ TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STable tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); - return (TsdbQueryHandleT)pQueryHandle; + return (TsdbQueryHandleT) pQueryHandle; +} + +TsdbQueryHandleT tsdbQueryLastRow(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList) { + STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList); + + pQueryHandle->type = TSDB_QUERY_TYPE_LAST; + pQueryHandle->order = TSDB_ORDER_DESC; + + changeQueryHandleForLastrowQuery(pQueryHandle); + return pQueryHandle; +} + +TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TsdbRepoT *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList) { + STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList); + + pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL; + pQueryHandle->order = TSDB_ORDER_ASC; + +// changeQueryHandleForLastrowQuery(pQueryHandle); + return pQueryHandle; } static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { @@ -236,8 +258,8 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { SDataRow row = SL_GET_NODE_DATA(node); pCheckInfo->lastKey = dataRowKey(row); // first timestamp in buffer - uTrace("%p uid:%" PRId64", tid:%d check data in buffer from skey:%" PRId64 ", order:%d", pHandle, - pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pCheckInfo->lastKey, pHandle->order); + uTrace("%p uid:%" PRId64", tid:%d check data in buffer from skey:%" PRId64 ", order:%d, %p", pHandle, + pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pCheckInfo->lastKey, pHandle->order, pHandle->qinfo); // all data in mem are checked already. if ((pCheckInfo->lastKey > pHandle->window.ekey && ASCENDING_ORDER_TRAVERSE(pHandle->order)) || @@ -293,8 +315,6 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo tsdbSetAndOpenHelperFile(&pQueryHandle->rhelper, fileGroup); // load all the comp offset value for all tables in this file - // tsdbLoadCompIdx(fileGroup, pQueryHandle->compIndex, 10000); // todo set dynamic max tables - *numOfBlocks = 0; size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); @@ -358,7 +378,7 @@ static SDataBlockInfo getTrueDataBlockInfo(STableCheckInfo* pCheckInfo, SCompBlo .window = {.skey = pBlock->keyFirst, .ekey = pBlock->keyLast}, .numOfCols = pBlock->numOfCols, .rows = pBlock->numOfPoints, - .sid = pCheckInfo->tableId.tid, + .tid = pCheckInfo->tableId.tid, .uid = pCheckInfo->tableId.uid, }; @@ -407,16 +427,11 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo SArray* sa = getDefaultLoadColumns(pQueryHandle, true); if (pCheckInfo->pDataCols == NULL) { - pCheckInfo->pDataCols = tdNewDataCols(1000, 2, 4096); + pCheckInfo->pDataCols = tdNewDataCols(1000, 100, 4096); //todo fix me } tdInitDataCols(pCheckInfo->pDataCols, tsdbGetTableSchema(tsdbGetMeta(pQueryHandle->pTsdb), pCheckInfo->pTableObj)); - // SFile* pFile = &pQueryHandle->pFileGroup->files[TSDB_FILE_TYPE_DATA]; - // if (pFile->fd == FD_INITIALIZER) { - // pFile->fd = open(pFile->fname, O_RDONLY); - // } - if (tsdbLoadBlockData(&(pQueryHandle->rhelper), pBlock, NULL) == 0) { SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo; @@ -552,7 +567,6 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf SQueryFilePos* cur = &pQueryHandle->cur; SDataBlockInfo blockInfo = getTrueDataBlockInfo(pCheckInfo, pBlock); -// pQueryHandle->rhelper.pDataCols[0]->cols[0]; SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; int32_t endPos = cur->pos; @@ -588,19 +602,33 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf } int32_t start = MIN(cur->pos, endPos); + +// if (start > 0) { +// tdPopDataColsPoints(pQueryHandle->rhelper.pDataCols[0], start); +// } // move the data block in the front to data block if needed - int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); - - for (int32_t i = 0; i < taosArrayGetSize(sa); ++i) { - int16_t colId = *(int16_t*)taosArrayGet(sa, i); + int32_t numOfCols = pQueryHandle->rhelper.pDataCols[0]->numOfCols; + int32_t reqCols = taosArrayGetSize(pQueryHandle->pColumns); + + for (int32_t i = 0; i < reqCols; ++i) { +// int16_t colId = *(int16_t*)taosArrayGet(sa, i); + SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); + int32_t bytes = pCol->info.bytes; for (int32_t j = 0; j < numOfCols; ++j) { - SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, j); - - if (pCol->info.colId == colId) { - memmove(pCol->pData, pQueryHandle->rhelper.pDataCols[0]->cols[i].pData + pCol->info.bytes * start, - pQueryHandle->realNumOfRows * pCol->info.bytes); + SDataCol* src = &pQueryHandle->rhelper.pDataCols[0]->cols[j]; + + if (pCol->info.colId == src->colId) { + if (pCol->info.type != TSDB_DATA_TYPE_BINARY && pCol->info.type != TSDB_DATA_TYPE_NCHAR) { + memmove(pCol->pData, src->pData + bytes * start, bytes * pQueryHandle->realNumOfRows); + } else { // handle the var-string + for(int32_t k = start; k < pQueryHandle->realNumOfRows + start; ++k) { + char* p = tdGetColDataOfRow(src, k); + memcpy(pCol->pData + k * bytes, p, varDataTLen(p)); // todo refactor + } + } + break; } } @@ -673,7 +701,7 @@ int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { } static void cleanBlockOrderSupporter(SBlockOrderSupporter* pSupporter, int32_t numOfTables) { - tfree(pSupporter->numOfBlocksPerMeter); + tfree(pSupporter->numOfBlocksPerTable); tfree(pSupporter->blockIndexArray); for (int32_t i = 0; i < numOfTables; ++i) { @@ -692,10 +720,10 @@ static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void* int32_t leftTableBlockIndex = pSupporter->blockIndexArray[leftTableIndex]; int32_t rightTableBlockIndex = pSupporter->blockIndexArray[rightTableIndex]; - if (leftTableBlockIndex > pSupporter->numOfBlocksPerMeter[leftTableIndex]) { + if (leftTableBlockIndex > pSupporter->numOfBlocksPerTable[leftTableIndex]) { /* left block is empty */ return 1; - } else if (rightTableBlockIndex > pSupporter->numOfBlocksPerMeter[rightTableIndex]) { + } else if (rightTableBlockIndex > pSupporter->numOfBlocksPerTable[rightTableIndex]) { /* right block is empty */ return -1; } @@ -727,11 +755,11 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO SBlockOrderSupporter sup = {0}; sup.numOfTables = numOfTables; - sup.numOfBlocksPerMeter = calloc(1, sizeof(int32_t) * numOfTables); + sup.numOfBlocksPerTable = calloc(1, sizeof(int32_t) * numOfTables); sup.blockIndexArray = calloc(1, sizeof(int32_t) * numOfTables); sup.pDataBlockInfo = calloc(1, POINTER_BYTES * numOfTables); - if (sup.numOfBlocksPerMeter == NULL || sup.blockIndexArray == NULL || sup.pDataBlockInfo == NULL) { + if (sup.numOfBlocksPerTable == NULL || sup.blockIndexArray == NULL || sup.pDataBlockInfo == NULL) { cleanBlockOrderSupporter(&sup, 0); return TSDB_CODE_SERV_OUT_OF_MEMORY; } @@ -745,7 +773,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO } SCompBlock* pBlock = pTableCheck->pCompInfo->blocks; - sup.numOfBlocksPerMeter[numOfQualTables] = pTableCheck->numOfBlocks; + sup.numOfBlocksPerTable[numOfQualTables] = pTableCheck->numOfBlocks; char* buf = calloc(1, sizeof(STableBlockInfo) * pTableCheck->numOfBlocks); if (buf == NULL) { @@ -763,7 +791,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO pBlockInfoEx->pTableCheckInfo = pTableCheck; // pBlockInfoEx->groupIdx = pTableCheckInfo[j]->groupIdx; // set the group index - // pBlockInfoEx->blockIndex = pTableCheckInfo[j]->start + k; // set the block index in original meter + // pBlockInfoEx->blockIndex = pTableCheckInfo[j]->start + k; // set the block index in original table cnt++; } @@ -772,7 +800,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO uTrace("%p create data blocks info struct completed, %d blocks in %d tables", pQueryHandle, cnt, numOfQualTables); - assert(cnt <= numOfBlocks && numOfQualTables <= numOfTables); // the pMeterDataInfo[j]->numOfBlocks may be 0 + assert(cnt <= numOfBlocks && numOfQualTables <= numOfTables); // the pTableQueryInfo[j]->numOfBlocks may be 0 sup.numOfTables = numOfQualTables; SLoserTreeInfo* pTree = NULL; @@ -792,8 +820,8 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO pQueryHandle->pDataBlockInfo[numOfTotal++] = pBlocksInfoEx[index]; // set data block index overflow, in order to disable the offset comparator - if (sup.blockIndexArray[pos] >= sup.numOfBlocksPerMeter[pos]) { - sup.blockIndexArray[pos] = sup.numOfBlocksPerMeter[pos] + 1; + if (sup.blockIndexArray[pos] >= sup.numOfBlocksPerTable[pos]) { + sup.blockIndexArray[pos] = sup.numOfBlocksPerTable[pos] + 1; } tLoserTreeAdjust(pTree, pos + sup.numOfTables); @@ -827,10 +855,14 @@ static bool getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle) { break; } - assert(numOfBlocks >= 0); uTrace("%p %d blocks found in file for %d table(s), fid:%d", pQueryHandle, numOfBlocks, numOfTables, pQueryHandle->pFileGroup->fileId); + assert(numOfBlocks >= 0); + if (numOfBlocks == 0) { + continue; + } + // todo return error code to query engine if (createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks) != TSDB_CODE_SUCCESS) { break; @@ -941,7 +973,60 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pqHandle) { } } -static int tsdbReadRowsFromCache(SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, TSKEY* skey, TSKEY* ekey, +void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle) { + STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pqHandle; + assert(!ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)); + + // starts from the buffer in case of descending timestamp order check data blocks + + // todo consider the query time window, current last_row does not apply the query time window + size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + + TSKEY key = 0; + int32_t index = -1; + + for(int32_t i = 0; i < numOfTables; ++i) { + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + if (pCheckInfo->pTableObj->lastKey > key) { //todo lastKey should not be 0 by default + key = pCheckInfo->pTableObj->lastKey; + index = i; + } + } + + // todo, there are no data in all the tables. opt performance + if (index == -1) { + return; + } + + // erase all other elements in array list, todo refactor + size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + for (int32_t i = 0; i < size; ++i) { + if (i == index) { + continue; + } + + STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + tSkipListDestroyIter(pTableCheckInfo->iter); + + if (pTableCheckInfo->pDataCols != NULL) { + tfree(pTableCheckInfo->pDataCols->buf); + } + + tfree(pTableCheckInfo->pDataCols); + tfree(pTableCheckInfo->pCompInfo); + } + + STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, index); + taosArrayDestroy(pQueryHandle->pTableCheckInfo); + + pQueryHandle->pTableCheckInfo = taosArrayInit(1, sizeof(STableCheckInfo)); + taosArrayPush(pQueryHandle->pTableCheckInfo, &info); + + // update the query time window according to the chosen last timestamp + pQueryHandle->window = (STimeWindow) {key, key}; +} + +static int tsdbReadRowsFromCache(SSkipListIterator* pIter, STable* pTable, TSKEY maxKey, int maxRowsToRead, TSKEY* skey, TSKEY* ekey, STsdbQueryHandle* pQueryHandle) { int numOfRows = 0; int32_t numOfCols = taosArrayGetSize(pQueryHandle->pColumns); @@ -971,8 +1056,11 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, TSKEY maxKey, int max *ekey = dataRowKey(row); - int32_t offset = 0; + int32_t offset = -1; char* pData = NULL; + + STSchema* pSchema = tsdbGetTableSchema(tsdbGetMeta(pQueryHandle->pTsdb), pTable); + int32_t numOfTableCols = schemaNCols(pSchema); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); @@ -983,8 +1071,25 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, TSKEY maxKey, int max pData = pColInfo->pData + (maxRowsToRead - numOfRows - 1) * pColInfo->info.bytes; } - memcpy(pData, dataRowTuple(row) + offset, pColInfo->info.bytes); - offset += pColInfo->info.bytes; + for(int32_t j = 0; j < numOfTableCols; ++j) { + if (pColInfo->info.colId == pSchema->columns[j].colId) { + offset = pSchema->columns[j].offset; + break; + } + } + + assert(offset != -1); // todo handle error + + if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { + void *value = tdGetRowDataOfCol(row, pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + offset); + memcpy(pData, value, varDataTLen(value)); + + offset += sizeof(int32_t); + } else { + memcpy(pData, dataRowTuple(row) + offset, pColInfo->info.bytes); + offset += pColInfo->info.bytes; + } + } numOfRows++; @@ -1050,7 +1155,7 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle) { if (pTable->mem != NULL) { // create mem table iterator if it is not created yet assert(pCheckInfo->iter != NULL); - rows = tsdbReadRowsFromCache(pCheckInfo->iter, pHandle->window.ekey, 4000, &skey, &ekey, pHandle); + rows = tsdbReadRowsFromCache(pCheckInfo->iter, pCheckInfo->pTableObj, pHandle->window.ekey, 4000, &skey, &ekey, pHandle); // update the last key value pCheckInfo->lastKey = ekey + step; @@ -1059,7 +1164,7 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle) { SDataBlockInfo blockInfo = { .uid = pTable->tableId.uid, - .sid = pTable->tableId.tid, + .tid = pTable->tableId.tid, .rows = rows, .window = {.skey = MIN(skey, ekey), .ekey = MAX(skey, ekey)} }; @@ -1110,10 +1215,6 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { } } -int32_t tsdbResetQuery(TsdbQueryHandleT* pQueryHandle, STimeWindow* window, TsdbPosT position, int16_t order) { - return 0; -} - SArray* tsdbRetrieveDataRow(TsdbQueryHandleT* pQueryHandle, SArray* pIdList, SQueryRowCond* pCond) { return NULL; } TsdbQueryHandleT* tsdbQueryFromTagConds(STsdbQueryCond* pCond, int16_t stableId, const char* pTagFilterStr) { @@ -1122,16 +1223,13 @@ TsdbQueryHandleT* tsdbQueryFromTagConds(STsdbQueryCond* pCond, int16_t stableId, SArray* tsdbGetTableList(TsdbQueryHandleT* pQueryHandle) { return NULL; } -static int32_t getAllTableIdList(STsdbRepo* tsdb, int64_t uid, SArray* list) { - STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); - assert(pTable != NULL); // assert pTable is a super table - - SSkipListIterator* iter = tSkipListCreateIter(pTable->pIndex); +static int32_t getAllTableIdList(STable* pSuperTable, SArray* list) { + SSkipListIterator* iter = tSkipListCreateIter(pSuperTable->pIndex); while (tSkipListIterNext(iter)) { SSkipListNode* pNode = tSkipListIterGet(iter); STable* t = *(STable**)SL_GET_NODE_DATA(pNode); - taosArrayPush(list, &t); + taosArrayPush(list, &t->tableId); } tSkipListDestroyIter(iter); @@ -1156,7 +1254,7 @@ static void convertQueryResult(SArray* pRes, SArray* pTableList) { size_t size = taosArrayGetSize(pTableList); for (int32_t i = 0; i < size; ++i) { STable* pTable = taosArrayGetP(pTableList, i); - taosArrayPush(pRes, &pTable); + taosArrayPush(pRes, &pTable->tableId); } } @@ -1165,27 +1263,31 @@ static void destroyHelper(void* param) { return; } + tQueryInfo* pInfo = (tQueryInfo*)param; - tVariantDestroy(&(pInfo->q)); + if (pInfo->optr != TSDB_RELATION_IN) { + tfree(pInfo->q); + } + +// tVariantDestroy(&(pInfo->q)); free(param); } -static void getTagColumnInfo(SExprTreeSupporter* pSupporter, SSchema* pSchema, int32_t* index) { - *index = 0; - +static int32_t getTagColumnInfo(SExprTreeSupporter* pSupporter, SSchema* pSchema) { // filter on table name(TBNAME) if (strcasecmp(pSchema->name, TSQL_TBNAME_L) == 0) { - *index = TSDB_TBNAME_COLUMN_INDEX; - return; + return TSDB_TBNAME_COLUMN_INDEX; } - while ((*index) < pSupporter->numOfTags) { - if (pSupporter->pTagSchema[*index].bytes == pSchema->bytes && - pSupporter->pTagSchema[*index].type == pSchema->type && - pSupporter->pTagSchema[*index].colId == pSchema->colId) { - break; + for(int32_t i = 0; i < pSupporter->numOfTags; ++i) { + if (pSupporter->pTagSchema[i].bytes == pSchema->bytes && + pSupporter->pTagSchema[i].type == pSchema->type && + pSupporter->pTagSchema[i].colId == pSchema->colId) { + return i; } } + + return -2; } void filterPrepare(void* expr, void* param) { @@ -1194,55 +1296,29 @@ void filterPrepare(void* expr, void* param) { return; } - int32_t i = 0, offset = 0; + int32_t i = 0; pExpr->_node.info = calloc(1, sizeof(tQueryInfo)); - tQueryInfo* pInfo = pExpr->_node.info; - SExprTreeSupporter* pSupporter = (SExprTreeSupporter*)param; - tVariant* pCond = pExpr->_node.pRight->pVal; - SSchema* pSchema = pExpr->_node.pLeft->pSchema; + tQueryInfo* pInfo = pExpr->_node.info; + tVariant* pCond = pExpr->_node.pRight->pVal; + SSchema* pSchema = pExpr->_node.pLeft->pSchema; - getTagColumnInfo(pSupporter, pSchema, &i); - assert((i >= 0 && i < TSDB_MAX_TAGS) || (i == TSDB_TBNAME_COLUMN_INDEX)); - assert((offset >= 0 && offset < TSDB_MAX_TAGS_LEN) || (offset == TSDB_TBNAME_COLUMN_INDEX)); + // todo : if current super table does not change schema yet, this function may failed, add test case + int32_t index = getTagColumnInfo(pSupporter, pSchema); + assert((index >= 0 && i < TSDB_MAX_TAGS) || (index == TSDB_TBNAME_COLUMN_INDEX)); pInfo->sch = *pSchema; - pInfo->colIndex = i; + pInfo->colIndex = index; pInfo->optr = pExpr->_node.optr; - pInfo->compare = getComparFunc(pSchema->type, pCond->nType, pInfo->optr); - - tVariantAssign(&pInfo->q, pCond); - if (pInfo->optr != TSDB_RELATION_IN) { - tVariantTypeSetType(&pInfo->q, pInfo->sch.type); - } -} - -int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) { - switch (type) { - case TSDB_DATA_TYPE_INT: DEFAULT_COMP(GET_INT32_VAL(f1), GET_INT32_VAL(f2)); - case TSDB_DATA_TYPE_DOUBLE: DEFAULT_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2)); - case TSDB_DATA_TYPE_FLOAT: DEFAULT_COMP(GET_FLOAT_VAL(f1), GET_FLOAT_VAL(f2)); - case TSDB_DATA_TYPE_BIGINT: DEFAULT_COMP(GET_INT64_VAL(f1), GET_INT64_VAL(f2)); - case TSDB_DATA_TYPE_SMALLINT: DEFAULT_COMP(GET_INT16_VAL(f1), GET_INT16_VAL(f2)); - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_BOOL: DEFAULT_COMP(GET_INT8_VAL(f1), GET_INT8_VAL(f2)); - case TSDB_DATA_TYPE_NCHAR: { - int32_t ret = wcsncmp((wchar_t*) f1, (wchar_t*) f2, size/TSDB_NCHAR_SIZE); - if (ret == 0) { - return ret; - } - return (ret < 0) ? -1 : 1; - } - default: { - int32_t ret = strncmp(f1, f2, (size_t)size); - if (ret == 0) { - return ret; - } - - return (ret < 0) ? -1 : 1; - } + pInfo->compare = getComparFunc(pSchema->type, pInfo->optr); + + if (pInfo->optr == TSDB_RELATION_IN) { + pInfo->q = (char*) pCond->arr; + } else { + pInfo->q = calloc(1, pSchema->bytes); + tVariantDump(pCond, pInfo->q, pSchema->type); } } @@ -1250,28 +1326,32 @@ typedef struct STableGroupSupporter { int32_t numOfCols; SColIndex* pCols; STSchema* pTagSchema; + void* tsdbMeta; } STableGroupSupporter; int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { STableGroupSupporter* pTableGroupSupp = (STableGroupSupporter*) param; + STableId* id1 = (STableId*) p1; + STableId* id2 = (STableId*) p2; - STable *pTable1 = *(STable **) p1; - STable *pTable2 = *(STable **) p2; + STable *pTable1 = tsdbGetTableByUid(pTableGroupSupp->tsdbMeta, id1->uid); + STable *pTable2 = tsdbGetTableByUid(pTableGroupSupp->tsdbMeta, id2->uid); for (int32_t i = 0; i < pTableGroupSupp->numOfCols; ++i) { SColIndex* pColIndex = &pTableGroupSupp->pCols[i]; int32_t colIndex = pColIndex->colIndex; - assert(colIndex >= 0 && colIndex < schemaNCols(pTableGroupSupp->pTagSchema)); + assert((colIndex >= 0 && colIndex < schemaNCols(pTableGroupSupp->pTagSchema)) || + (colIndex == TSDB_TBNAME_COLUMN_INDEX)); char * f1 = NULL; char * f2 = NULL; int32_t type = 0; int32_t bytes = 0; - if (colIndex == -1) { // todo fix me, table name -// f1 = s1->tags; -// f2 = s2->tags; + if (colIndex == TSDB_TBNAME_COLUMN_INDEX) { + f1 = pTable1->name; + f2 = pTable2->name; type = TSDB_DATA_TYPE_BINARY; bytes = TSDB_TABLE_NAME_LEN; } else { @@ -1293,34 +1373,36 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { return 0; } -void createTableGroupImpl(SArray* pGroups, STable** pTables, size_t numOfTables, STableGroupSupporter* pSupp, __ext_compar_fn_t compareFn) { - SArray* g = taosArrayInit(16, sizeof(SPair)); +void createTableGroupImpl(SArray* pGroups, SArray* pTableIdList, size_t numOfTables, STableGroupSupporter* pSupp, + __ext_compar_fn_t compareFn) { + STableId* pId = taosArrayGet(pTableIdList, 0); - SPair p = {.first = pTables[0]}; - taosArrayPush(g, &p); + SArray* g = taosArrayInit(16, sizeof(STableId)); + taosArrayPush(g, pId); for (int32_t i = 1; i < numOfTables; ++i) { - int32_t ret = compareFn(&pTables[i - 1], &pTables[i], pSupp); + STableId* prev = taosArrayGet(pTableIdList, i - 1); + STableId* p = taosArrayGet(pTableIdList, i); + + int32_t ret = compareFn(prev, p, pSupp); assert(ret == 0 || ret == -1); if (ret == 0) { - SPair p1 = {.first = pTables[i]}; - taosArrayPush(g, &p1); + taosArrayPush(g, p); } else { taosArrayPush(pGroups, &g); // current group is ended, start a new group - g = taosArrayInit(16, POINTER_BYTES); - - SPair p1 = {.first = pTables[i]}; - taosArrayPush(g, &p1); + g = taosArrayInit(16, sizeof(STableId)); + + taosArrayPush(g, p); } } taosArrayPush(pGroups, &g); } -SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols) { +SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols, + TsdbRepoT* tsdb) { assert(pTableList != NULL); - SArray* pTableGroup = taosArrayInit(1, POINTER_BYTES); size_t size = taosArrayGetSize(pTableList); @@ -1330,23 +1412,23 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC } if (numOfOrderCols == 0 || size == 1) { // no group by tags clause or only one table - SArray* sa = taosArrayInit(size, sizeof(SPair)); + SArray* sa = taosArrayInit(size, sizeof(STableId)); for(int32_t i = 0; i < size; ++i) { - STable* pTable = taosArrayGetP(pTableList, i); - SPair p = {.first = pTable}; - taosArrayPush(sa, &p); + STableId* tableId = taosArrayGet(pTableList, i); + taosArrayPush(sa, tableId); } taosArrayPush(pTableGroup, &sa); uTrace("all %d tables belong to one group", size); } else { STableGroupSupporter *pSupp = (STableGroupSupporter *) calloc(1, sizeof(STableGroupSupporter)); + pSupp->tsdbMeta = tsdbGetMeta(tsdb); pSupp->numOfCols = numOfOrderCols; pSupp->pTagSchema = pTagSchema; pSupp->pCols = pCols; - taosqsort(pTableList->pData, size, POINTER_BYTES, pSupp, tableGroupComparFn); - createTableGroupImpl(pTableGroup, pTableList->pData, size, pSupp, tableGroupComparFn); + taosqsort(pTableList->pData, size, sizeof(STableId), pSupp, tableGroupComparFn); + createTableGroupImpl(pTableGroup, pTableList, size, pSupp, tableGroupComparFn); tfree(pSupp); } @@ -1360,7 +1442,7 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { char* val = NULL; int8_t type = pInfo->sch.type; - + if (pInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) { val = pTable->name; type = TSDB_DATA_TYPE_BINARY; @@ -1371,14 +1453,14 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { int32_t ret = 0; if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { if (pInfo->optr == TSDB_RELATION_IN) { - ret = pInfo->compare(val, pInfo->q.arr); + ret = pInfo->compare(val, pInfo->q); } else { - ret = pInfo->compare(val, pInfo->q.pz); + ret = pInfo->compare(val, pInfo->q); } } else { - tVariant t = {0}; - tVariantCreateFromBinary(&t, val, (uint32_t)pInfo->sch.bytes, type); - ret = pInfo->compare(&t.i64Key, &pInfo->q.i64Key); +// tVariant t = {0}; +// tVariantCreateFromBinary(&t, val, (uint32_t)pInfo->sch.bytes, type); + ret = pInfo->compare(val, pInfo->q); } switch (pInfo->optr) { @@ -1410,17 +1492,17 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { default: assert(false); } + return true; } - static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) { // query according to the binary expression STSchema* pSchema = pSTable->tagSchema; SSchema* schema = calloc(schemaNCols(pSchema), sizeof(SSchema)); for (int32_t i = 0; i < schemaNCols(pSchema); ++i) { schema[i].colId = schemaColAt(pSchema, i)->colId; - schema[i].type = schemaColAt(pSchema, i)->type; + schema[i].type = schemaColAt(pSchema, i)->type; schema[i].bytes = schemaColAt(pSchema, i)->bytes; } @@ -1441,39 +1523,34 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) return TSDB_CODE_SUCCESS; } - -int32_t tsdbQueryByTagsCond( - TsdbRepoT *tsdb, - int64_t uid, - const char *pTagCond, - size_t len, - int16_t tagNameRelType, - const char* tbnameCond, - STableGroupInfo *pGroupInfo, - SColIndex *pColIndex, - int32_t numOfCols -) { - STable* pSTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); - if (pSTable == NULL) { - uError("failed to get stable, uid:%" PRIu64, uid); +int32_t tsdbQuerySTableByTagCond(TsdbRepoT *tsdb, int64_t uid, const char *pTagCond, size_t len, int16_t tagNameRelType, + const char* tbnameCond, STableGroupInfo *pGroupInfo, SColIndex *pColIndex, int32_t numOfCols) { + STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); + if (pTable == NULL) { + uError("failed to get stable, uid:%, %p" PRIu64, uid); return TSDB_CODE_INVALID_TABLE_ID; } - SArray* res = taosArrayInit(8, POINTER_BYTES); - STSchema* pTagSchema = tsdbGetTableTagSchema(tsdbGetMeta(tsdb), pSTable); + SArray* res = taosArrayInit(8, sizeof(STableId)); + STSchema* pTagSchema = tsdbGetTableTagSchema(tsdbGetMeta(tsdb), pTable); // no tags and tbname condition, all child tables of this stable are involved if (tbnameCond == NULL && (pTagCond == NULL || len == 0)) { - int32_t ret = getAllTableIdList(tsdb, uid, res); + int32_t ret = getAllTableIdList(pTable, res); if (ret == TSDB_CODE_SUCCESS) { pGroupInfo->numOfTables = taosArrayGetSize(res); - pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); + pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, tsdb); + + uTrace("no tbname condition or tagcond, all tables belongs to one group, numOfTables:%d", pGroupInfo->numOfTables); + } else { + // todo add error } + taosArrayDestroy(res); return ret; } - int32_t ret = TSDB_CODE_SUCCESS; + int32_t ret = TSDB_CODE_SUCCESS; tExprNode* expr = NULL; TRY(32) { @@ -1504,9 +1581,9 @@ int32_t tsdbQueryByTagsCond( // TODO: more error handling } END_TRY - doQueryTableList(pSTable, res, expr); + doQueryTableList(pTable, res, expr); pGroupInfo->numOfTables = taosArrayGetSize(res); - pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); + pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, tsdb); taosArrayDestroy(res); return ret; @@ -1522,9 +1599,9 @@ int32_t tsdbGetOneTableGroup(TsdbRepoT* tsdb, int64_t uid, STableGroupInfo* pGro pGroupInfo->numOfTables = 1; pGroupInfo->pGroupList = taosArrayInit(1, POINTER_BYTES); - SArray* group = taosArrayInit(1, POINTER_BYTES); + SArray* group = taosArrayInit(1, sizeof(STableId)); - taosArrayPush(group, &pTable); + taosArrayPush(group, &pTable->tableId); taosArrayPush(pGroupInfo->pGroupList, &group); return TSDB_CODE_SUCCESS; @@ -1546,12 +1623,10 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { } tfree(pTableCheckInfo->pDataCols); - tfree(pTableCheckInfo->pCompInfo); } taosArrayDestroy(pQueryHandle->pTableCheckInfo); - tfree(pQueryHandle->compIndex); size_t cols = taosArrayGetSize(pQueryHandle->pColumns); for (int32_t i = 0; i < cols; ++i) { diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h index dbb74edf07d7535d1de4d1485fdd3df410b82c5d..7edd032034c18a3120647525f38ed64e86d0cd08 100644 --- a/src/util/inc/tarray.h +++ b/src/util/inc/tarray.h @@ -98,13 +98,13 @@ void taosArrayRemove(SArray* pArray, size_t index); * @param pDst * @param pSrc */ -void taosArrayCopy(SArray* pDst, SArray* pSrc); +void taosArrayCopy(SArray* pDst, const SArray* pSrc); /** * clone a new array * @param pSrc */ -SArray* taosArrayClone(SArray* pSrc); +SArray* taosArrayClone(const SArray* pSrc); /** * destroy array list diff --git a/src/util/inc/tcache.h b/src/util/inc/tcache.h index a29e720e67904b5ffbc32a575ccb86dd0ccdbabb..2e26f8f0e3f83a4a2d315b9dafc262e8ace4ba7a 100644 --- a/src/util/inc/tcache.h +++ b/src/util/inc/tcache.h @@ -69,6 +69,7 @@ typedef struct { SHashObj * pHashTable; int numOfElemsInTrash; // number of element in trash int16_t deleting; // set the deleting flag to stop refreshing ASAP. + T_REF_DECLARE() #if defined(LINUX) pthread_rwlock_t lock; diff --git a/src/util/inc/tcompare.h b/src/util/inc/tcompare.h index 3484b5b8baf7819811e19346ff7c8d1a211d7a41..8aaa39e483389f021fecf456fa15adf5bc93da42 100644 --- a/src/util/inc/tcompare.h +++ b/src/util/inc/tcompare.h @@ -38,9 +38,11 @@ int patternMatch(const char *zPattern, const char *zString, size_t size, const S int WCSPatternMatch(const wchar_t *zPattern, const wchar_t *zString, size_t size, const SPatternCompareInfo *pInfo); +int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size); + __compar_fn_t getKeyComparFunc(int32_t keyType); -__compar_fn_t getComparFunc(int32_t type, int32_t filterDataType, int32_t optr); +__compar_fn_t getComparFunc(int32_t type, int32_t optr); #ifdef __cplusplus } diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h index 55f4496755673d40cd3e4141e59ffbbb1dddf07f..b03e0a1c5b5cb72a7741cae4b233c16db6c6bebf 100644 --- a/src/util/inc/tutil.h +++ b/src/util/inc/tutil.h @@ -110,11 +110,6 @@ extern "C" { #define POW2(x) ((x) * (x)) -typedef struct SPair { - void* first; - void* sec; -} SPair; - int32_t strdequote(char *src); void strtrim(char *src); @@ -149,7 +144,7 @@ void getTmpfilePath(const char *fileNamePattern, char *dstPath); int32_t taosInitTimer(void (*callback)(int), int32_t ms); void taosUninitTimer(); -bool taosMbsToUcs4(char *mbs, int32_t mbs_len, char *ucs4, int32_t ucs4_max_len); +bool taosMbsToUcs4(char *mbs, int32_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t* len); int tasoUcs4Compare(void* f1_ucs4, void *f2_ucs4, int bytes); diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index 51684e767cd517bda0d7b69e04e2fc3c3fac4acd..5ef4417710adbaa7151ee19b2f276653d69a3067 100755 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -143,7 +143,7 @@ void taosArrayRemove(SArray* pArray, size_t index) { pArray->size -= 1; } -void taosArrayCopy(SArray* pDst, SArray* pSrc) { +void taosArrayCopy(SArray* pDst, const SArray* pSrc) { assert(pSrc != NULL && pDst != NULL); if (pDst->capacity < pSrc->size) { @@ -162,7 +162,7 @@ void taosArrayCopy(SArray* pDst, SArray* pSrc) { pDst->size = pSrc->size; } -SArray* taosArrayClone(SArray* pSrc) { +SArray* taosArrayClone(const SArray* pSrc) { assert(pSrc != NULL); if (pSrc->size == 0) { // empty array list @@ -176,7 +176,6 @@ SArray* taosArrayClone(SArray* pSrc) { return dst; } - void taosArrayDestroy(SArray* pArray) { if (pArray == NULL) { return; diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index e1e76173e62333fa3f4a4b5b6b0f019668242b39..63fe124de28b982c46305af652eb2cad3c841754 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -79,7 +79,7 @@ static FORCE_INLINE void taosFreeNode(void *data) { */ static SCacheDataNode *taosCreateHashNode(const char *key, size_t keyLen, const char *pData, size_t size, uint64_t duration) { - size_t totalSize = size + sizeof(SCacheDataNode) + keyLen; + size_t totalSize = size + sizeof(SCacheDataNode) + keyLen + 1; SCacheDataNode *pNewNode = calloc(1, totalSize); if (pNewNode == NULL) { @@ -324,13 +324,26 @@ static void doCleanupDataCache(SCacheObj *pCacheObj) { static void taosCacheRefresh(void *handle, void *tmrId) { SCacheObj *pCacheObj = (SCacheObj *)handle; - if (pCacheObj == NULL || taosHashGetSize(pCacheObj->pHashTable) == 0) { + if (pCacheObj == NULL || T_REF_VAL_GET(pCacheObj) == 0) { uTrace("object is destroyed. no refresh retry"); return; } - if (pCacheObj->deleting == 1) { - doCleanupDataCache(pCacheObj); + int16_t ref = T_REF_INC(pCacheObj); + if (ref == 1) { + T_REF_DEC(pCacheObj); + return; + } + + // todo add the ref before start the timer + int32_t num = taosHashGetSize(pCacheObj->pHashTable); + if (num == 0) { + ref = T_REF_DEC(pCacheObj); + if (ref == 0) { + doCleanupDataCache(pCacheObj); + } else { + taosTmrReset(taosCacheRefresh, pCacheObj->refreshTime, pCacheObj, pCacheObj->tmrCtrl, &pCacheObj->pTimer); + } return; } @@ -355,13 +368,16 @@ static void taosCacheRefresh(void *handle, void *tmrId) { __cache_unlock(pCacheObj); taosHashDestroyIter(pIter); - - if (pCacheObj->deleting == 1) { // clean up resources and abort - doCleanupDataCache(pCacheObj); - } else { + taosTrashEmpty(pCacheObj, false); - taosTmrReset(taosCacheRefresh, pCacheObj->refreshTime, pCacheObj, pCacheObj->tmrCtrl, &pCacheObj->pTimer); - } + + ref = T_REF_DEC(pCacheObj); + if (ref == 0) { + doCleanupDataCache(pCacheObj); + return; + } else { + taosTmrReset(taosCacheRefresh, pCacheObj->refreshTime, pCacheObj, pCacheObj->tmrCtrl, &pCacheObj->pTimer); + } } SCacheObj *taosCacheInit(void *tmrCtrl, int64_t refreshTime) { @@ -399,6 +415,7 @@ SCacheObj *taosCacheInit(void *tmrCtrl, int64_t refreshTime) { return NULL; } + T_REF_INC(pCacheObj); return pCacheObj; } @@ -552,5 +569,8 @@ void taosCacheCleanup(SCacheObj *pCacheObj) { return; } - pCacheObj->deleting = 1; + int32_t ref = T_REF_DEC(pCacheObj); + if (ref == 0) { + doCleanupDataCache(pCacheObj); + } } diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index 4d78a8cf0adada9841bd70b006d5ceb588faa4b6..13a5a8580e8f09602ce545595fd24e2423c52179 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -216,7 +216,7 @@ static UNUSED_FUNC int32_t compareStrPatternComp(const void* pLeft, const void* } static int32_t compareFindStrInArray(const void* pLeft, const void* pRight) { - const SArray* arr = (const SArray*)pRight; + const SArray* arr = (const SArray*) pRight; return taosArraySearchString(arr, pLeft) == NULL ? 0 : 1; } @@ -232,55 +232,42 @@ static UNUSED_FUNC int32_t compareWStrPatternComp(const void* pLeft, const void* } // todo promote the type definition before the comparsion -__compar_fn_t getComparFunc(int32_t type, int32_t filterDataType, int32_t optr) { +__compar_fn_t getComparFunc(int32_t type, int32_t optr) { __compar_fn_t comparFn = NULL; switch (type) { - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_SMALLINT: { + comparFn = compareInt16Val; break; + } + + case TSDB_DATA_TYPE_INT: { + comparFn = compareInt32Val; break; + } + case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: { -// assert(type == filterDataType); - if (filterDataType == TSDB_DATA_TYPE_BIGINT || filterDataType == TSDB_DATA_TYPE_TIMESTAMP) { - comparFn = compareInt64Val; - } else if (filterDataType >= TSDB_DATA_TYPE_FLOAT && filterDataType <= TSDB_DATA_TYPE_DOUBLE) { - comparFn = compareIntDoubleVal; - } - - break; + comparFn = compareInt64Val; break; } - case TSDB_DATA_TYPE_BOOL: { - if (filterDataType >= TSDB_DATA_TYPE_BOOL && filterDataType <= TSDB_DATA_TYPE_BIGINT) { - comparFn = compareInt32Val; - } else if (filterDataType >= TSDB_DATA_TYPE_FLOAT && filterDataType <= TSDB_DATA_TYPE_DOUBLE) { - comparFn = compareIntDoubleVal; - } - break; + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT:{ + comparFn = compareInt8Val; break; } - case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_FLOAT: { + comparFn = compareDoubleVal; break; + } + case TSDB_DATA_TYPE_DOUBLE: { - if (filterDataType >= TSDB_DATA_TYPE_BOOL && filterDataType <= TSDB_DATA_TYPE_BIGINT) { - comparFn = compareDoubleIntVal; - } else if (filterDataType >= TSDB_DATA_TYPE_FLOAT && filterDataType <= TSDB_DATA_TYPE_DOUBLE) { - comparFn = compareDoubleVal; - } - break; + comparFn = compareDoubleVal; break; } case TSDB_DATA_TYPE_BINARY: { if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ - assert(filterDataType == TSDB_DATA_TYPE_BINARY); comparFn = compareStrPatternComp; - } else if (optr == TSDB_RELATION_IN) { - assert(filterDataType == TSDB_DATA_TYPE_ARRAY); comparFn = compareFindStrInArray; - } else { /* normal relational comparFn */ - assert(filterDataType == TSDB_DATA_TYPE_BINARY); comparFn = compareStrVal; } @@ -288,8 +275,6 @@ __compar_fn_t getComparFunc(int32_t type, int32_t filterDataType, int32_t optr) } case TSDB_DATA_TYPE_NCHAR: { - assert(filterDataType == TSDB_DATA_TYPE_NCHAR); - if (optr == TSDB_RELATION_LIKE) { comparFn = compareWStrPatternComp; } else { @@ -348,3 +333,30 @@ __compar_fn_t getKeyComparFunc(int32_t keyType) { return comparFn; } + +int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) { + switch (type) { + case TSDB_DATA_TYPE_INT: DEFAULT_COMP(GET_INT32_VAL(f1), GET_INT32_VAL(f2)); + case TSDB_DATA_TYPE_DOUBLE: DEFAULT_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2)); + case TSDB_DATA_TYPE_FLOAT: DEFAULT_COMP(GET_FLOAT_VAL(f1), GET_FLOAT_VAL(f2)); + case TSDB_DATA_TYPE_BIGINT: DEFAULT_COMP(GET_INT64_VAL(f1), GET_INT64_VAL(f2)); + case TSDB_DATA_TYPE_SMALLINT: DEFAULT_COMP(GET_INT16_VAL(f1), GET_INT16_VAL(f2)); + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_BOOL: DEFAULT_COMP(GET_INT8_VAL(f1), GET_INT8_VAL(f2)); + case TSDB_DATA_TYPE_NCHAR: { + int32_t ret = wcsncmp((wchar_t*) f1, (wchar_t*) f2, size/TSDB_NCHAR_SIZE); + if (ret == 0) { + return ret; + } + return (ret < 0) ? -1 : 1; + } + default: { + int32_t ret = strncmp(f1, f2, (size_t)size); + if (ret == 0) { + return ret; + } + + return (ret < 0) ? -1 : 1; + } + } +} diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 109b4b45b6fa56975ed31622a912332ede276531..4149449e6cf8ae4c036297aa11ee22cfd37492cc 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -325,7 +325,7 @@ SArray* tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey, int16_t keyType) { pSkipList->state.queryCount++; #endif - __compar_fn_t filterComparFn = getComparFunc(pSkipList->keyInfo.type, keyType, 0); + __compar_fn_t filterComparFn = getComparFunc(pSkipList->keyInfo.type, 0); int32_t ret = -1; for (int32_t i = sLevel; i >= 0; --i) { SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, i); @@ -389,7 +389,7 @@ SSkipListIterator *tSkipListCreateIterFromVal(SSkipList* pSkipList, const char* SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; int32_t ret = -1; - __compar_fn_t filterComparFn = getComparFunc(pSkipList->keyInfo.type, type, 0); + __compar_fn_t filterComparFn = getKeyComparFunc(pSkipList->keyInfo.type); SSkipListNode* pNode = pSkipList->pHead; for (int32_t i = pSkipList->level - 1; i >= 0; --i) { diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 707d6741f78ac40e500d6168e7925b8fbc836bce..396f6f1ef889f390d0449c5d44add78b1a30b6ba 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -490,7 +490,7 @@ bool taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) { #endif } -bool taosMbsToUcs4(char *mbs, int32_t mbs_len, char *ucs4, int32_t ucs4_max_len) { +bool taosMbsToUcs4(char *mbs, int32_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t* len) { memset(ucs4, 0, ucs4_max_len); #ifdef USE_LIBICONV iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset); @@ -501,6 +501,10 @@ bool taosMbsToUcs4(char *mbs, int32_t mbs_len, char *ucs4, int32_t ucs4_max_len) return false; } iconv_close(cd); + if (len != NULL) { + *len = outLen; + } + return true; #else mbstate_t state = {0}; diff --git a/src/util/tests/cacheTest.cpp b/src/util/tests/cacheTest.cpp index 13fb7d7afbdcd669bf107f5869ff7842b34ceee0..b636bcfac0c1252d087f570b0c99a2a2f7ec812b 100644 --- a/src/util/tests/cacheTest.cpp +++ b/src/util/tests/cacheTest.cpp @@ -25,7 +25,7 @@ TEST(testCase, client_cache_test) { const char* key1 = "test1"; char data1[] = "test11"; - char* cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data1, strlen(data1), 1); + char* cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data1, strlen(data1)+1, 1); sleep(REFRESH_TIME_IN_SEC+1); printf("obj is still valid: %s\n", cachedObj); @@ -34,19 +34,17 @@ TEST(testCase, client_cache_test) { taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false); /* the object is cleared by cache clean operation */ - cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data2, strlen(data2), 20); + cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data2, strlen(data2)+1, 20); printf("after updated: %s\n", cachedObj); printf("start to remove data from cache\n"); taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false); printf("end of removing data from cache\n"); - getchar(); - const char* key3 = "test2"; const char* data3 = "kkkkkkk"; - char* cachedObj2 = (char*) taosCachePut(tscCacheHandle, key3, data3, strlen(data3), 1); + char* cachedObj2 = (char*) taosCachePut(tscCacheHandle, key3, data3, strlen(data3) + 1, 1); printf("%s\n", cachedObj2); taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false); @@ -57,18 +55,18 @@ TEST(testCase, client_cache_test) { char key5[] = "test5"; char data5[] = "data5kkkkk"; - cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data5, strlen(data5), 20); + cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data5, strlen(data5) + 1, 20); const char* data6= "new Data after updated"; taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false); - cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data6, strlen(data6), 20); + cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data6, strlen(data6) + 1, 20); printf("%s\n", cachedObj2); taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true); const char* data7 = "add call update procedure"; - cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data7, strlen(data7), 20); + cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data7, strlen(data7) + 1, 20); printf("%s\n=======================================\n\n", cachedObj2); char* cc = (char*) taosCacheAcquireByName(tscCacheHandle, key5); @@ -137,6 +135,4 @@ TEST(testCase, cache_resize_test) { printf("retrieve %d object cost:%" PRIu64 " us,avg:%f\n", num, endTime - startTime, (endTime - startTime)/(double)num); taosCacheCleanup(pCache); - taosMsleep(20000); - getchar(); } \ No newline at end of file diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 201214ded46c3a0f7da349a89fa5324dbe6eac0a..b111f56e390cd1e69ed8d10591686db39b7ea44a 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -56,7 +56,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, void *pCont, int32_t cont qinfo_t pQInfo = NULL; if (contLen != 0) { - pRet->code = qCreateQueryInfo(pVnode->tsdb, pQueryTableMsg, &pQInfo); + pRet->code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo); SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp)); pRsp->qhandle = htobe64((uint64_t) (pQInfo)); @@ -97,7 +97,7 @@ static int32_t vnodeProcessRetrieveMsg(SVnodeObj *pVnode, void *pCont, int32_t c if (qHasMoreResultsToRetrieve(pQInfo)) { pRet->qhandle = pQInfo; code = TSDB_CODE_ACTION_NEED_REPROCESSED; - } else { + } else { // no further execution invoked, release the ref to vnode qDestroyQueryInfo(pQInfo); vnodeRelease(pVnode); diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index cc13d9d02624f50671c82fb44ac56cc7236b5133..2f49280cb6af32fc308a59a7820bcde8993f444f 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -136,7 +136,8 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe } tsdbTableSetTagSchema(&tCfg, pDestTagSchema, false); tsdbTableSetSName(&tCfg, pTable->superTableId, false); - + tsdbTableSetSuperUid(&tCfg, htobe64(pTable->superTableUid)); + char *pTagData = pTable->data + totalCols * sizeof(SSchema); int accumBytes = 0; SDataRow dataRow = tdNewDataRowFromSchema(pDestTagSchema); diff --git a/tests/examples/c/demo.c b/tests/examples/c/demo.c index e5a70740f112f44428c3a75d96c0bba3927d8731..a2073b09b32a922cd097369b912a4fbddbec5e93 100644 --- a/tests/examples/c/demo.c +++ b/tests/examples/c/demo.c @@ -75,7 +75,15 @@ int main(int argc, char *argv[]) { doQuery(taos, "create database if not exists test"); doQuery(taos, "use test"); - doQuery(taos, "select count(*),k,sum(k) from m1 group by k"); + doQuery(taos, "select * from t1 order by ts desc"); + +// doQuery(taos, "create table t1(ts timestamp, k binary(12), f nchar(2))"); +// for(int32_t i = 0; i< 100000; ++i) { +// doQuery(taos, "select m1.ts,m1.a from m1, m2 where m1.ts=m2.ts and m1.a=m2.b;"); +// usleep(500000); +// } + +// doQuery(taos, "insert into tm0 values('2020-1-1 1:1:1', 'abc')"); // doQuery(taos, "create table if not exists tm0 (ts timestamp, k int);"); // doQuery(taos, "insert into tm0 values('2020-1-1 1:1:1', 1);"); // doQuery(taos, "insert into tm0 values('2020-1-1 1:1:2', 2);");