diff --git a/CMakeLists.txt b/CMakeLists.txt index 553da9245bc5d805b9a95cc2120d6b6783da2b30..41231f053bdf416119876f7501ffdcf9b17b4231 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -296,5 +296,6 @@ ENDIF () ADD_SUBDIRECTORY(deps) ADD_SUBDIRECTORY(src) +ADD_SUBDIRECTORY(tests) INCLUDE(CPack) diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index a64f0581fb01ee6093e3fdc284030e0303f59060..55fa45475a658105c33ae620cd8a5d922a838466 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -6,6 +6,7 @@ INCLUDE_DIRECTORIES(jni) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc) INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) AUX_SOURCE_DIRECTORY(src SRC) diff --git a/src/client/inc/tscJoinProcess.h b/src/client/inc/tscSubquery.h similarity index 89% rename from src/client/inc/tscJoinProcess.h rename to src/client/inc/tscSubquery.h index bc1101df276c4d4ffe52143fb79bbd0acaae2ec8..f8a6fbf5b11d688b16bf7625771b1e893bed52c1 100644 --- a/src/client/inc/tscJoinProcess.h +++ b/src/client/inc/tscSubquery.h @@ -32,6 +32,12 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code); SJoinSubquerySupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, int32_t index); void tscDestroyJoinSupporter(SJoinSubquerySupporter* pSupporter); +int32_t tscHandleMasterJoinQuery(SSqlObj* pSql); + +int32_t tscHandleMasterSTableQuery(SSqlObj *pSql); + +int32_t tscHandleMultivnodeInsert(SSqlObj *pSql); + #ifdef __cplusplus } #endif diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index eaefbdd85f7451a4387a067ec826e6e0f838d503..7ec3bc679efc8131d9cad592a0cbd99c941286ab 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -29,11 +29,11 @@ extern "C" { #include "tscSecondaryMerge.h" #include "tsclient.h" -#define UTIL_METER_IS_SUPERTABLE(metaInfo) \ - (((metaInfo)->pMeterMeta != NULL) && ((metaInfo)->pMeterMeta->tableType == TSDB_SUPER_TABLE)) -#define UTIL_METER_IS_NOMRAL_METER(metaInfo) (!(UTIL_METER_IS_SUPERTABLE(metaInfo))) -#define UTIL_METER_IS_CREATE_FROM_METRIC(metaInfo) \ - (((metaInfo)->pMeterMeta != NULL) && ((metaInfo)->pMeterMeta->tableType == TSDB_CHILD_TABLE)) +#define UTIL_TABLE_IS_SUPERTABLE(metaInfo) \ + (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_SUPER_TABLE)) +#define UTIL_TABLE_IS_NOMRAL_TABLE(metaInfo) (!(UTIL_TABLE_IS_SUPERTABLE(metaInfo))) +#define UTIL_TABLE_CREATE_FROM_STABLE(metaInfo) \ + (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_CHILD_TABLE)) #define TSDB_COL_IS_TAG(f) (((f)&TSDB_COL_TAG) != 0) @@ -67,7 +67,7 @@ typedef struct SJoinSubquerySupporter { } SJoinSubquerySupporter; int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, const char* name, - STableMeta* pMeterMeta, STableDataBlocks** dataBlocks); + STableMeta* pTableMeta, STableDataBlocks** dataBlocks); void tscAppendDataBlock(SDataBlockList* pList, STableDataBlocks* pBlocks); void tscDestroyDataBlock(STableDataBlocks* pDataBlock); @@ -81,7 +81,7 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock); void tscFreeUnusedDataBlocks(SDataBlockList* pList); int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pDataList); int32_t tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList, int64_t id, int32_t size, - int32_t startOffset, int32_t rowSize, const char* tableId, STableMeta* pMeterMeta, + int32_t startOffset, int32_t rowSize, const char* tableId, STableMeta* pTableMeta, STableDataBlocks** dataBlocks); SVnodeSidList* tscGetVnodeSidList(SSuperTableMeta* pMetricmeta, int32_t vnodeIdx); @@ -104,7 +104,7 @@ bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex); bool tscProjectionQueryOnTable(SQueryInfo* pQueryInfo); -bool tscIsTwoStageMergeMetricQuery(SQueryInfo* pQueryInfo, int32_t tableIndex); +bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex); bool tscQueryOnMetric(SSqlCmd* pCmd); bool tscQueryMetricTags(SQueryInfo* pQueryInfo); bool tscIsSelectivityWithTagQuery(SSqlCmd* pCmd); @@ -114,7 +114,7 @@ void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex void addRequiredTagColumn(SQueryInfo* pQueryInfo, int32_t tagColIndex, int32_t tableIndex); -int32_t setMeterID(SMeterMetaInfo* pMeterMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql); +int32_t setMeterID(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql); void tscClearInterpInfo(SQueryInfo* pQueryInfo); bool tscIsInsertOrImportData(char* sqlstr); @@ -173,7 +173,7 @@ int32_t tscValidateName(SSQLToken* pToken); void tscIncStreamExecutionCount(void* pStream); -bool tscValidateColumnId(SMeterMetaInfo* pMeterMetaInfo, int32_t colId); +bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId); // get starter position of metric query condition (query on tags) in SSqlCmd.payload SCond* tsGetMetricQueryCondPos(STagCond* pCond, uint64_t tableIndex); @@ -190,26 +190,26 @@ void tscCleanSqlCmd(SSqlCmd* pCmd); bool tscShouldFreeAsyncSqlObj(SSqlObj* pSql); void tscRemoveAllMeterMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache); -SMeterMetaInfo* tscGetMeterMetaInfo(SSqlCmd *pCmd, int32_t subClauseIndex, int32_t tableIndex); -SMeterMetaInfo* tscGetMeterMetaInfoFromQueryInfo(SQueryInfo *pQueryInfo, int32_t tableIndex); +STableMetaInfo* tscGetTableMetaInfoFromCmd(SSqlCmd *pCmd, int32_t subClauseIndex, int32_t tableIndex); +STableMetaInfo* tscGetMetaInfo(SQueryInfo *pQueryInfo, int32_t tableIndex); SQueryInfo *tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex); int32_t tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex, SQueryInfo** pQueryInfo); -SMeterMetaInfo* tscGetMeterMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, int32_t* index); -void tscClearMeterMetaInfo(SMeterMetaInfo* pMeterMetaInfo, bool removeFromCache); +STableMetaInfo* tscGetMeterMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, int32_t* index); +void tscClearMeterMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache); -SMeterMetaInfo* tscAddMeterMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pMeterMeta, SSuperTableMeta* pMetricMeta, +STableMetaInfo* tscAddMeterMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, SSuperTableMeta* pMetricMeta, int16_t numOfTags, int16_t* tags); -SMeterMetaInfo* tscAddEmptyMeterMetaInfo(SQueryInfo *pQueryInfo); +STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo *pQueryInfo); int32_t tscAddSubqueryInfo(SSqlCmd *pCmd); void tscFreeSubqueryInfo(SSqlCmd* pCmd); void tscClearSubqueryInfo(SSqlCmd* pCmd); void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* keyStr, uint64_t uid); int tscGetMetricMeta(SSqlObj* pSql, int32_t clauseIndex); -int tscGetMeterMeta(SSqlObj* pSql, SMeterMetaInfo* pMeterMetaInfo); -int tscGetMeterMetaEx(SSqlObj* pSql, SMeterMetaInfo* pMeterMetaInfo, bool createIfNotExists); +int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo); +int tscGetMeterMetaEx(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, bool createIfNotExists); void tscResetForNextRetrieve(SSqlRes* pRes); @@ -252,8 +252,6 @@ void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp); void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows); void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)()); -int32_t launchMultivnodeInsert(SSqlObj *pSql); - #ifdef __cplusplus } #endif diff --git a/src/util/inc/tschemautil.h b/src/client/inc/tschemautil.h similarity index 52% rename from src/util/inc/tschemautil.h rename to src/client/inc/tschemautil.h index 64bbf94f4245f7c9886dec9514ff11a101872517..2fc77d69bbe0f276aa5b963da95ef8e6c98dcdae 100644 --- a/src/util/inc/tschemautil.h +++ b/src/client/inc/tschemautil.h @@ -20,13 +20,56 @@ extern "C" { #endif -#include #include "taosmsg.h" #include "tstoken.h" +#include "tsclient.h" #define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS) -struct SSchema; +//struct SSchema; + +/** + * get the number of tags of this table + * @param pTableMeta + * @return + */ +int32_t tscGetNumOfTags(const STableMeta* pTableMeta); + +/** + * get the number of columns of this table + * @param pTableMeta + * @return + */ +int32_t tscGetNumOfColumns(const STableMeta* pTableMeta); + +/** + * get the basic info of this table + * @param pTableMeta + * @return + */ +STableComInfo tscGetTableInfo(const STableMeta* pTableMeta); + +/** + * get the schema + * @param pTableMeta + * @return + */ +SSchema* tscGetTableSchema(const STableMeta* pTableMeta); + +/** + * get the tag schema + * @param pMeta + * @return + */ +SSchema *tscGetTableTagSchema(const STableMeta *pMeta); + +/** + * + * @param pMeta + * @param startCol + * @return + */ +SSchema *tscGetTableColumnSchema(const STableMeta *pMeta, int32_t startCol); /** * check if the schema is valid or not, including following aspects: @@ -42,21 +85,23 @@ struct SSchema; */ bool isValidSchema(struct SSchema *pSchema, int32_t numOfCols); -struct SSchema *tsGetSchema(STableMeta *pMeta); - -struct SSchema *tsGetTagSchema(STableMeta *pMeta); +/** + * get the schema for the "tbname" column. it is a built column + * @return + */ +SSchema tscGetTbnameColumnSchema(); -struct SSchema *tsGetColumnSchema(STableMeta *pMeta, int32_t startCol); -struct SSchema tsGetTbnameColumnSchema(); +/** + * create the table meta from the msg + * @param pTableMetaMsg + * @param size size of the table meta + * @return + */ +STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size); +//todo tags value as well as the table id structure needs refactor char *tsGetTagsValue(STableMeta *pMeta); -bool tsMeterMetaIdentical(STableMeta *p1, STableMeta *p2); - -void extractTableName(char *tableId, char *name); - -SSQLToken extractDBName(char *tableId, char *name); - void extractTableNameFromToken(SSQLToken *pToken, SSQLToken* pTable); #ifdef __cplusplus diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 329c5743708b150eb1a733d4592c1cc6bca319de..611f52676ef2354ad89984a55855fae264fd42ca 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -21,17 +21,19 @@ extern "C" { #endif #include "os.h" + +#include "qsqlparser.h" +#include "qsqltype.h" +#include "qtsbuf.h" #include "taos.h" +#include "taosdef.h" #include "taosmsg.h" #include "tglobalcfg.h" #include "tlog.h" -#include "taosdef.h" +#include "trpc.h" #include "tsqlfunction.h" #include "tutil.h" -#include "trpc.h" -#include "qsqltype.h" -#include "qsqlparser.h" -#include "qtsbuf.h" +#include "tarray.h" #define TSC_GET_RESPTR_BASE(res, _queryinfo, col) (res->data + ((_queryinfo)->fieldsInfo.pSqlExpr[col]->offset) * res->numOfRows) @@ -46,8 +48,28 @@ typedef struct SSqlGroupbyExpr { int16_t orderType; // order by type: asc/desc } SSqlGroupbyExpr; -typedef struct SMeterMetaInfo { - STableMeta * pMeterMeta; // metermeta +typedef struct STableComInfo { + uint8_t numOfTags; + uint8_t precision; + int16_t numOfColumns; + int16_t rowSize; +} STableComInfo; + +typedef struct STableMeta { + //super table if it is created according to super table, otherwise, tableInfo is used + union { struct STableMeta* pSTable; STableComInfo tableInfo; }; + uint8_t tableType; + int8_t numOfVpeers; + int16_t sversion; + SVnodeDesc vpeerDesc[TSDB_VNODES_SUPPORT]; + int32_t vgid; // virtual group id, which current table belongs to + int32_t sid; // the index of one table in a virtual node + uint64_t uid; // unique id of a table + SSchema schema[]; // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info +} STableMeta; + +typedef struct STableMetaInfo { + STableMeta * pTableMeta; // table meta, cached in client side and acquried by name SSuperTableMeta *pMetricMeta; // metricmeta /* @@ -55,14 +77,14 @@ typedef struct SMeterMetaInfo { * 2. keep the vnode index for multi-vnode insertion */ int32_t vnodeIndex; - char name[TSDB_TABLE_ID_LEN + 1]; // table(super table) name + char name[TSDB_TABLE_ID_LEN]; // (super) table name int16_t numOfTags; // total required tags in query, including groupby tags int16_t tagColumnIndex[TSDB_MAX_TAGS]; // clause + tag projection -} SMeterMetaInfo; +} STableMetaInfo; /* the structure for sql function in select clause */ typedef struct SSqlExpr { - char aliasName[TSDB_COL_NAME_LEN + 1]; // as aliasName + char aliasName[TSDB_COL_NAME_LEN]; // as aliasName SColIndexEx colInfo; int64_t uid; // refactor use the pointer int16_t functionId; // function id in aAgg array @@ -83,7 +105,6 @@ typedef struct SFieldInfo { int16_t numOfOutputCols; // number of column in result int16_t numOfAlloc; // allocated size TAOS_FIELD *pFields; -// short * pOffset; /* * define if this column is belong to the queried result, it may be add by parser to faciliate @@ -174,7 +195,7 @@ typedef struct STableDataBlocks { * the metermeta for current table, the metermeta will be used during submit stage, keep a ref * to avoid it to be removed from cache */ - STableMeta *pMeterMeta; + STableMeta *pTableMeta; union { char *filename; @@ -199,7 +220,7 @@ typedef struct SDataBlockList { typedef struct SQueryInfo { int16_t command; // the command may be different for each subclause, so keep it seperately. uint16_t type; // query/insert/import type - char intervalTimeUnit; + char slidingTimeUnit; int64_t etime, stime; int64_t intervalTime; // aggregation time interval @@ -215,7 +236,7 @@ typedef struct SQueryInfo { SOrderVal order; int16_t interpoType; // interpolate type int16_t numOfTables; - SMeterMetaInfo **pMeterInfo; + STableMetaInfo **pTableMetaInfo; struct STSBuf * tsBuf; int64_t * defaultVal; // default value for interpolation char * msg; // pointer to the pCmd->payload to keep error message temporarily @@ -238,7 +259,7 @@ typedef struct { union { bool existsCheck; // check if the table exists or not bool inStream; // denote if current sql is executed in stream or not - bool createOnDemand; // if the table is missing, on-the-fly create it. during getmeterMeta + bool autoCreated; // if the table is missing, on-the-fly create it. during getmeterMeta int8_t dataSourceType; // load data from file or not }; @@ -296,7 +317,7 @@ typedef struct { struct SLocalReducer *pLocalReducer; } SSqlRes; -typedef struct _tsc_obj { +typedef struct STscObj { void * signature; void * pTimer; char mgmtIp[TSDB_USER_LEN]; @@ -311,7 +332,7 @@ typedef struct _tsc_obj { struct SSqlObj *pSql; struct SSqlObj *pHb; struct SSqlObj *sqlList; - struct _sstream *streamList; + struct SSqlStream *streamList; pthread_mutex_t mutex; } STscObj; @@ -344,7 +365,7 @@ typedef struct SSqlObj { struct SSqlObj * prev, *next; } SSqlObj; -typedef struct _sstream { +typedef struct SSqlStream { SSqlObj *pSql; uint32_t streamId; char listed; @@ -369,7 +390,7 @@ typedef struct _sstream { void *param; void (*callback)(void *); // Callback function when stream is stopped from client level - struct _sstream *prev, *next; + struct SSqlStream *prev, *next; } SSqlStream; int32_t tscInitRpc(const char *user, const char *secret); @@ -377,14 +398,12 @@ int32_t tscInitRpc(const char *user, const char *secret); // tscSql API int tsParseSql(SSqlObj *pSql, bool multiVnodeInsertion); -void tscInitMsgs(); +void tscInitMsgsFp(); extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); void tscProcessMsgFromServer(SRpcMsg *rpcMsg); int tscProcessSql(SSqlObj *pSql); -void tscAsyncInsertMultiVnodesProxy(void *param, TAOS_RES *tres, int numOfRows); - int tscRenewMeterMeta(SSqlObj *pSql, char *tableId); void tscQueueAsyncRes(SSqlObj *pSql); diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index 228403c79d318d922f5571a9663b3c97bbffbbc8..dfaedc7c143890b7541d0eae59ece25af13730c4 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -13,13 +13,13 @@ * along with this program. If not, see . */ -#include "os.h" #include "com_taosdata_jdbc_TSDBJNIConnector.h" +#include "os.h" #include "taos.h" #include "tlog.h" -#include "tscJoinProcess.h" -#include "tsclient.h" +#include "tscSubquery.h" #include "tscUtil.h" +#include "tsclient.h" #include "ttime.h" int __init = 0; diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 77fe94210035b47c1e0eccc8ca54409a9e66920c..2bab6e03fe0ab4fd746d4c246299252b4f85fe44 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -25,6 +25,7 @@ #include "tutil.h" #include "tnote.h" #include "tsched.h" +#include "tschemautil.h" static void tscProcessFetchRow(SSchedMsg *pMsg); static void tscAsyncQueryRowsForNextVnode(void *param, TAOS_RES *tres, int numOfRows); @@ -83,7 +84,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const } // TODO return the correct error code to client in tscQueueAsyncError -void taos_query_a(TAOS *taos, const char *sqlstr, void (*fp)(void *, TAOS_RES *, int), void *param) { +void taos_query_a(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param) { STscObj *pObj = (STscObj *)taos; if (pObj == NULL || pObj->signature != pObj) { tscError("bug!!! pObj:%p", pObj); @@ -396,51 +397,9 @@ void tscQueueAsyncFreeResult(SSqlObj *pSql) { taosScheduleTask(tscQhandle, &schedMsg); } -void tscAsyncInsertMultiVnodesProxy(void *param, TAOS_RES *tres, int numOfRows) { - SSqlObj *pSql = (SSqlObj *)param; - SSqlCmd *pCmd = &pSql->cmd; - int32_t code = TSDB_CODE_SUCCESS; - - assert(pCmd->dataSourceType != 0 && pSql->signature == pSql); - - int32_t index = 0; - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, index); - - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - assert(pQueryInfo->numOfTables == 1 || pQueryInfo->numOfTables == 2); - - SDataBlockList *pDataBlocks = pCmd->pDataBlocks; - if (pDataBlocks == NULL || pMeterMetaInfo->vnodeIndex >= pDataBlocks->nSize) { - // restore user defined fp - pSql->fp = pSql->fetchFp; - tscTrace("%p Async insertion completed, destroy data block list", pSql); - - // release data block data - pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); - - // all data has been sent to vnode, call user function - (*pSql->fp)(pSql->param, tres, numOfRows); - } else { - do { - code = tscCopyDataBlockToPayload(pSql, pDataBlocks->pData[pMeterMetaInfo->vnodeIndex++]); - if (code != TSDB_CODE_SUCCESS) { - tscTrace("%p prepare submit data block failed in async insertion, vnodeIdx:%d, total:%d, code:%d", - pSql, pMeterMetaInfo->vnodeIndex - 1, pDataBlocks->nSize, code); - } - - } while (code != TSDB_CODE_SUCCESS && pMeterMetaInfo->vnodeIndex < pDataBlocks->nSize); - - // build submit msg may fail - if (code == TSDB_CODE_SUCCESS) { - tscTrace("%p async insertion, vnodeIdx:%d, total:%d", pSql, pMeterMetaInfo->vnodeIndex - 1, pDataBlocks->nSize); - tscProcessSql(pSql); - } - } -} - int tscSendMsgToServer(SSqlObj *pSql); -void tscMeterMetaCallBack(void *param, TAOS_RES *res, int code) { +void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { SSqlObj *pSql = (SSqlObj *)param; if (pSql == NULL || pSql->signature != pSql) return; @@ -464,10 +423,10 @@ void tscMeterMetaCallBack(void *param, TAOS_RES *res, int code) { tscTrace("%p renew tableMeta successfully, command:%d, code:%d, retry:%d", pSql, pSql->cmd.command, pSql->res.code, pSql->retry); - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, 0, 0); - assert(pMeterMetaInfo->pMeterMeta == NULL); + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); + assert(pTableMetaInfo->pTableMeta == NULL); - tscGetMeterMeta(pSql, pMeterMetaInfo); + tscGetTableMeta(pSql, pTableMetaInfo); code = tscSendMsgToServer(pSql); if (code != 0) { pRes->code = code; @@ -489,18 +448,18 @@ void tscMeterMetaCallBack(void *param, TAOS_RES *res, int code) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); if ((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_SUBQUERY) == TSDB_QUERY_TYPE_STABLE_SUBQUERY) { - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - assert(pMeterMetaInfo->pMeterMeta->numOfTags != 0 && pMeterMetaInfo->vnodeIndex >= 0 && pSql->param != NULL); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0) && pTableMetaInfo->vnodeIndex >= 0 && pSql->param != NULL); SRetrieveSupport *trs = (SRetrieveSupport *)pSql->param; SSqlObj * pParObj = trs->pParentSqlObj; - assert(pParObj->signature == pParObj && trs->subqueryIndex == pMeterMetaInfo->vnodeIndex && - pMeterMetaInfo->pMeterMeta->numOfTags != 0); + assert(pParObj->signature == pParObj && trs->subqueryIndex == pTableMetaInfo->vnodeIndex && + tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0); tscTrace("%p get metricMeta during super table query successfully", pSql); - code = tscGetMeterMeta(pSql, pMeterMetaInfo); + code = tscGetTableMeta(pSql, pTableMetaInfo); pRes->code = code; if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; @@ -513,11 +472,11 @@ void tscMeterMetaCallBack(void *param, TAOS_RES *res, int code) { if (pCmd->isParseFinish) { tscTrace("%p resend data to vnode in metermeta callback since sql has been parsed completed", pSql); - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); - code = tscGetMeterMeta(pSql, pMeterMetaInfo); + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + code = tscGetTableMeta(pSql, pTableMetaInfo); assert(code == TSDB_CODE_SUCCESS); - if (pMeterMetaInfo->pMeterMeta) { + if (pTableMetaInfo->pTableMeta) { code = tscSendMsgToServer(pSql); if (code == TSDB_CODE_SUCCESS) return; } @@ -528,13 +487,13 @@ void tscMeterMetaCallBack(void *param, TAOS_RES *res, int code) { } } else { // stream computing - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); - code = tscGetMeterMeta(pSql, pMeterMetaInfo); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + code = tscGetTableMeta(pSql, pTableMetaInfo); pRes->code = code; if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; - if (code == TSDB_CODE_SUCCESS && UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) { + if (code == TSDB_CODE_SUCCESS && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { code = tscGetMetricMeta(pSql, pCmd->clauseIndex); pRes->code = code; diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 885b78010d388a70513f5e9234b67fdf82d36c14..689f9715d08b4569817d16daca224c2482c7643c 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -24,7 +24,7 @@ #include "taosdef.h" #include "taosmsg.h" #include "tlog.h" -#include "tscJoinProcess.h" +#include "tscSubquery.h" #include "tscompression.h" #include "tsqlfunction.h" #include "ttime.h" @@ -158,7 +158,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI pError("Illegal data type %d or data type length %d", dataType, dataBytes); return TSDB_CODE_INVALID_SQL; } - + if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_INTERP) { @@ -167,47 +167,47 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI *intermediateResBytes = *bytes + sizeof(SResultInfo); return TSDB_CODE_SUCCESS; } - + if (functionId == TSDB_FUNC_COUNT) { *type = TSDB_DATA_TYPE_BIGINT; *bytes = sizeof(int64_t); *intermediateResBytes = *bytes; return TSDB_CODE_SUCCESS; } - + if (functionId == TSDB_FUNC_ARITHM) { *type = TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(double); *intermediateResBytes = *bytes; return TSDB_CODE_SUCCESS; } - + if (functionId == TSDB_FUNC_TS_COMP) { *type = TSDB_DATA_TYPE_BINARY; *bytes = sizeof(int32_t); // this results is compressed ts data *intermediateResBytes = POINTER_BYTES; return TSDB_CODE_SUCCESS; } - + if (isSuperTable) { if (functionId == TSDB_FUNC_MIN || functionId == TSDB_FUNC_MAX) { *type = TSDB_DATA_TYPE_BINARY; *bytes = dataBytes + DATA_SET_FLAG_SIZE; *intermediateResBytes = *bytes; - + return TSDB_CODE_SUCCESS; } else if (functionId == TSDB_FUNC_SUM) { *type = TSDB_DATA_TYPE_BINARY; *bytes = sizeof(SSumInfo); *intermediateResBytes = *bytes; - + return TSDB_CODE_SUCCESS; } else if (functionId == TSDB_FUNC_AVG) { *type = TSDB_DATA_TYPE_BINARY; *bytes = sizeof(SAvgInfo); *intermediateResBytes = *bytes; return TSDB_CODE_SUCCESS; - + } else if (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE) { *type = TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(SRateInfo); @@ -217,25 +217,25 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI *type = TSDB_DATA_TYPE_BINARY; *bytes = sizeof(STopBotInfo) + (sizeof(tValuePair) + POINTER_BYTES + extLength) * param; *intermediateResBytes = *bytes; - + return TSDB_CODE_SUCCESS; } else if (functionId == TSDB_FUNC_SPREAD) { *type = TSDB_DATA_TYPE_BINARY; *bytes = sizeof(SSpreadInfo); *intermediateResBytes = *bytes; - + return TSDB_CODE_SUCCESS; } else if (functionId == TSDB_FUNC_APERCT) { *type = TSDB_DATA_TYPE_BINARY; *bytes = sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1) + sizeof(SHistogramInfo) + sizeof(SAPercentileInfo); *intermediateResBytes = *bytes; - + return TSDB_CODE_SUCCESS; } else if (functionId == TSDB_FUNC_LAST_ROW) { *type = TSDB_DATA_TYPE_BINARY; *bytes = sizeof(SLastrowInfo) + dataBytes; *intermediateResBytes = *bytes; - + return TSDB_CODE_SUCCESS; } else if (functionId == TSDB_FUNC_TWA) { *type = TSDB_DATA_TYPE_DOUBLE; @@ -244,14 +244,14 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI return TSDB_CODE_SUCCESS; } } - + if (functionId == TSDB_FUNC_SUM) { if (dataType >= TSDB_DATA_TYPE_TINYINT && dataType <= TSDB_DATA_TYPE_BIGINT) { *type = TSDB_DATA_TYPE_BIGINT; } else { *type = TSDB_DATA_TYPE_DOUBLE; } - + *bytes = sizeof(int64_t); *intermediateResBytes = sizeof(SSumInfo); return TSDB_CODE_SUCCESS; @@ -267,7 +267,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI *intermediateResBytes = sizeof(STwaInfo); return TSDB_CODE_SUCCESS; } - + if (functionId == TSDB_FUNC_AVG) { *type = TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(double); @@ -307,9 +307,9 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { *type = (int16_t)dataType; *bytes = (int16_t)dataBytes; - + size_t size = sizeof(STopBotInfo) + (sizeof(tValuePair) + POINTER_BYTES + extLength) * param; - + // the output column may be larger than sizeof(STopBotInfo) *intermediateResBytes = size; } else if (functionId == TSDB_FUNC_LAST_ROW) { @@ -319,7 +319,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI } else { return TSDB_CODE_INVALID_SQL; } - + return TSDB_CODE_SUCCESS; } @@ -335,20 +335,20 @@ void resetResultInfo(SResultInfo *pResInfo) { pResInfo->initialized = false; } void initResultInfo(SResultInfo *pResInfo) { pResInfo->initialized = true; // the this struct has been initialized flag - + pResInfo->complete = false; pResInfo->hasResult = false; pResInfo->numOfRes = 0; - + memset(pResInfo->interResultBuf, 0, (size_t)pResInfo->bufLen); } void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable) { assert(pResInfo->interResultBuf == NULL); - + pResInfo->bufLen = size; pResInfo->superTableQ = superTable; - + pResInfo->interResultBuf = calloc(1, (size_t)size); } @@ -363,9 +363,9 @@ static bool function_setup(SQLFunctionCtx *pCtx) { if (pResInfo->initialized) { return false; } - + memset(pCtx->aOutputBuf, 0, (size_t)pCtx->outputBytes); - + initResultInfo(pResInfo); return true; } @@ -379,7 +379,7 @@ static bool function_setup(SQLFunctionCtx *pCtx) { */ static void function_finalizer(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + if (pResInfo->hasResult != DATA_SET_FLAG) { pTrace("no result generated, result is set to NULL"); setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); @@ -388,50 +388,43 @@ static void function_finalizer(SQLFunctionCtx *pCtx) { doFinalizer(pCtx); } +static bool usePreVal(SQLFunctionCtx *pCtx) { + return pCtx->preAggVals.isSet && pCtx->size == pCtx->preAggVals.size; +} + /* * count function does need the finalize, if data is missing, the default value, which is 0, is used * count function does not use the pCtx->interResBuf to keep the intermediate buffer */ static void count_function(SQLFunctionCtx *pCtx) { int32_t numOfElem = 0; - - if (IS_DATA_BLOCK_LOADED(pCtx->blockStatus)) { - /* - * In following cases, the data block is loaded: - * 1. A first/last file block for a query - * 2. Required to handle other queries, such as apercentile/twa/stddev etc. - * 3. A cache block - */ + + /* + * 1. column data missing (schema modified) causes pCtx->hasNull == true. pCtx->preAggVals.isSet == true; + * 2. for general non-primary key columns, pCtx->hasNull may be true or false, pCtx->preAggVals.isSet == true; + * 3. for primary key column, pCtx->hasNull always be false, pCtx->preAggVals.isSet == false; + */ + if (usePreVal(pCtx)) { + numOfElem = pCtx->size - pCtx->preAggVals.statis.numOfNull; + } else { if (pCtx->hasNull) { for (int32_t i = 0; i < pCtx->size; ++i) { char *val = GET_INPUT_CHAR_INDEX(pCtx, i); if (isNull(val, pCtx->inputType)) { continue; } - + numOfElem += 1; } } else { numOfElem = pCtx->size; } - } else { - /* - * 1. column data missing (schema modified) causes pCtx->hasNull == true. pCtx->preAggVals.isSet == true; - * 2. for general non-primary key columns, pCtx->hasNull may be true or false, pCtx->preAggVals.isSet == true; - * 3. for primary key column, pCtx->hasNull always be false, pCtx->preAggVals.isSet == false; - */ - if (pCtx->preAggVals.isSet) { - numOfElem = pCtx->size - pCtx->preAggVals.numOfNull; - } else { - assert(pCtx->hasNull == false); - numOfElem = pCtx->size; - } } - + if (numOfElem > 0) { GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } - + *((int64_t *)pCtx->aOutputBuf) += numOfElem; SET_VAL(pCtx, numOfElem, 1); } @@ -441,11 +434,11 @@ static void count_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + SET_VAL(pCtx, 1, 1); - + *((int64_t *)pCtx->aOutputBuf) += 1; - + // do not need it actually SResultInfo *pInfo = GET_RES_INFO(pCtx); pInfo->hasResult = DATA_SET_FLAG; @@ -456,7 +449,7 @@ static void count_func_merge(SQLFunctionCtx *pCtx) { for (int32_t i = 0; i < pCtx->size; ++i) { *((int64_t *)pCtx->aOutputBuf) += pData[i]; } - + SET_VAL(pCtx, pCtx->size, 1); } @@ -469,7 +462,7 @@ static void count_func_merge(SQLFunctionCtx *pCtx) { * @param filterCols * @return */ -int32_t count_load_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId, int32_t blockStatus) { +int32_t count_load_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { if (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { return BLK_DATA_NO_NEEDED; } else { @@ -477,7 +470,7 @@ int32_t count_load_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32 } } -int32_t no_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId, int32_t blockStatus) { +int32_t no_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { return BLK_DATA_NO_NEEDED; } @@ -530,26 +523,26 @@ do { \ static void do_sum(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - + // Only the pre-computing information loaded and actual data does not loaded - if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus) && pCtx->preAggVals.isSet) { - notNullElems = pCtx->size - pCtx->preAggVals.numOfNull; - assert(pCtx->size >= pCtx->preAggVals.numOfNull); - + if (pCtx->preAggVals.isSet && pCtx->preAggVals.size == pCtx->size) { + notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull; + assert(pCtx->size >= pCtx->preAggVals.statis.numOfNull); + if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { int64_t *retVal = (int64_t*) pCtx->aOutputBuf; - *retVal += pCtx->preAggVals.sum; + *retVal += pCtx->preAggVals.statis.sum; } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { double *retVal = (double*) pCtx->aOutputBuf; - *retVal += GET_DOUBLE_VAL(&(pCtx->preAggVals.sum)); + *retVal += GET_DOUBLE_VAL(&(pCtx->preAggVals.statis.sum)); } } else { // computing based on the true data block void *pData = GET_INPUT_CHAR(pCtx); notNullElems = 0; - + if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { int64_t *retVal = (int64_t*) pCtx->aOutputBuf; - + if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { LIST_ADD_N(*retVal, pCtx, pData, int8_t, notNullElems, pCtx->inputType); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { @@ -567,10 +560,10 @@ static void do_sum(SQLFunctionCtx *pCtx) { LIST_ADD_N(*retVal, pCtx, pData, float, notNullElems, pCtx->inputType); } } - + // data in the check operation are all null, not output SET_VAL(pCtx, notNullElems, 1); - + if (notNullElems > 0) { GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } @@ -581,10 +574,10 @@ static void do_sum_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + SET_VAL(pCtx, 1, 1); int64_t *res = (int64_t*) pCtx->aOutputBuf; - + if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { *res += GET_INT8_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { @@ -600,13 +593,13 @@ static void do_sum_f(SQLFunctionCtx *pCtx, int32_t index) { double *retVal = (double*) pCtx->aOutputBuf; *retVal += GET_FLOAT_VAL(pData); } - + GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } static void sum_function(SQLFunctionCtx *pCtx) { do_sum(pCtx); - + // keep the result data in output buffer, not in the intermediate buffer SResultInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) { @@ -618,7 +611,7 @@ static void sum_function(SQLFunctionCtx *pCtx) { static void sum_function_f(SQLFunctionCtx *pCtx, int32_t index) { do_sum_f(pCtx, index); - + // keep the result data in output buffer, not in the intermediate buffer SResultInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) { @@ -629,20 +622,20 @@ static void sum_function_f(SQLFunctionCtx *pCtx, int32_t index) { static int32_t sum_merge_impl(const SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - + GET_TRUE_DATA_TYPE(); SResultInfo *pResInfo = GET_RES_INFO(pCtx); assert(pResInfo->superTableQ); - + for (int32_t i = 0; i < pCtx->size; ++i) { char * input = GET_INPUT_CHAR_INDEX(pCtx, i); SSumInfo *pInput = (SSumInfo *)input; if (pInput->hasResult != DATA_SET_FLAG) { continue; } - + notNullElems++; - + switch (type) { case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: @@ -657,16 +650,16 @@ static int32_t sum_merge_impl(const SQLFunctionCtx *pCtx) { } } } - + return notNullElems; } static void sum_func_merge(SQLFunctionCtx *pCtx) { int32_t notNullElems = sum_merge_impl(pCtx); - + SET_VAL(pCtx, notNullElems, 1); SSumInfo *pSumInfo = (SSumInfo *)pCtx->aOutputBuf; - + if (notNullElems > 0) { // pCtx->numOfIteratedElems += notNullElems; pSumInfo->hasResult = DATA_SET_FLAG; @@ -675,29 +668,29 @@ static void sum_func_merge(SQLFunctionCtx *pCtx) { static void sum_func_second_merge(SQLFunctionCtx *pCtx) { int32_t notNullElems = sum_merge_impl(pCtx); - + SET_VAL(pCtx, notNullElems, 1); SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + if (notNullElems > 0) { pResInfo->hasResult = DATA_SET_FLAG; } } -static int32_t precal_req_load_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId, int32_t blockStatus) { +static int32_t precal_req_load_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { return BLK_DATA_FILEDS_NEEDED; } -static int32_t data_req_load_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId, int32_t blockStatus) { +static int32_t data_req_load_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { return BLK_DATA_ALL_NEEDED; } // todo: if column in current data block are null, opt for this case -static int32_t first_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId, int32_t blockStatus) { +static int32_t first_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { if (pCtx->order == TSQL_SO_DESC) { return BLK_DATA_NO_NEEDED; } - + // no result for first query, data block is required if (GET_RES_INFO(pCtx)->numOfRes <= 0) { return BLK_DATA_ALL_NEEDED; @@ -706,11 +699,11 @@ static int32_t first_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, } } -static int32_t last_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId, int32_t blockStatus) { +static int32_t last_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { if (pCtx->order == TSQL_SO_ASC) { return BLK_DATA_NO_NEEDED; } - + if (GET_RES_INFO(pCtx)->numOfRes <= 0) { return BLK_DATA_ALL_NEEDED; } else { @@ -718,12 +711,11 @@ static int32_t last_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, } } -static int32_t first_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId, - int32_t blockStatus) { +static int32_t first_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { if (pCtx->order == TSQL_SO_DESC) { return BLK_DATA_NO_NEEDED; } - + // result buffer has not been set yet. return BLK_DATA_ALL_NEEDED; //todo optimize the filter info @@ -735,12 +727,11 @@ static int32_t first_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY // } } -static int32_t last_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId, - int32_t blockStatus) { +static int32_t last_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { if (pCtx->order == TSQL_SO_ASC) { return BLK_DATA_NO_NEEDED; } - + return BLK_DATA_ALL_NEEDED; // SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes); // if (pInfo->hasResult != DATA_SET_FLAG) { @@ -758,26 +749,26 @@ static int32_t last_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY */ static void avg_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - + // NOTE: keep the intermediate result into the interResultBuf SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf; double * pVal = &pAvgInfo->sum; - - if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus) && pCtx->preAggVals.isSet) { + + if (usePreVal(pCtx)) { // Pre-aggregation - notNullElems = pCtx->size - pCtx->preAggVals.numOfNull; + notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull; assert(notNullElems >= 0); - + if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { - *pVal += pCtx->preAggVals.sum; + *pVal += pCtx->preAggVals.statis.sum; } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - *pVal += GET_DOUBLE_VAL(&(pCtx->preAggVals.sum)); + *pVal += GET_DOUBLE_VAL(&(pCtx->preAggVals.statis.sum)); } } else { void *pData = GET_INPUT_CHAR(pCtx); - + if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { LIST_ADD_N(*pVal, pCtx, pData, int8_t, notNullElems, pCtx->inputType); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { @@ -792,18 +783,18 @@ static void avg_function(SQLFunctionCtx *pCtx) { LIST_ADD_N(*pVal, pCtx, pData, float, notNullElems, pCtx->inputType); } } - + if (!pCtx->hasNull) { assert(notNullElems == pCtx->size); } - + SET_VAL(pCtx, notNullElems, 1); pAvgInfo->num += notNullElems; - + if (notNullElems > 0) { pResInfo->hasResult = DATA_SET_FLAG; } - + // keep the data into the final output buffer for super table query since this execution may be the last one if (pResInfo->superTableQ) { memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SAvgInfo)); @@ -815,14 +806,14 @@ static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + SET_VAL(pCtx, 1, 1); - + // NOTE: keep the intermediate result into the interResultBuf SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf; - + if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { pAvgInfo->sum += GET_INT8_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { @@ -836,13 +827,13 @@ static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) { } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { pAvgInfo->sum += GET_FLOAT_VAL(pData); } - + // restore sum and count of elements pAvgInfo->num += 1; - + // set has result flag pResInfo->hasResult = DATA_SET_FLAG; - + // keep the data into the final output buffer for super table query since this execution may be the last one if (pResInfo->superTableQ) { memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SAvgInfo)); @@ -852,20 +843,20 @@ static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void avg_func_merge(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); assert(pResInfo->superTableQ); - + SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf; char * input = GET_INPUT_CHAR(pCtx); - + for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { SAvgInfo *pInput = (SAvgInfo *)input; if (pInput->num == 0) { // current buffer is null continue; } - + pAvgInfo->sum += pInput->sum; pAvgInfo->num += pInput->num; } - + // if the data set hasResult is not set, the result is null if (pAvgInfo->num > 0) { pResInfo->hasResult = DATA_SET_FLAG; @@ -875,18 +866,18 @@ static void avg_func_merge(SQLFunctionCtx *pCtx) { static void avg_func_second_merge(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + double *sum = (double*) pCtx->aOutputBuf; char * input = GET_INPUT_CHAR(pCtx); - + for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { SAvgInfo *pInput = (SAvgInfo *)input; if (pInput->num == 0) { // current input is null continue; } - + *sum += pInput->sum; - + // keep the number of data into the temp buffer *(int64_t *)pResInfo->interResultBuf += pInput->num; } @@ -897,29 +888,29 @@ static void avg_func_second_merge(SQLFunctionCtx *pCtx) { */ static void avg_finalizer(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - + if (GET_INT64_VAL(pResInfo->interResultBuf) <= 0) { setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); return; // empty table } - + *(double *)pCtx->aOutputBuf = (*(double *)pCtx->aOutputBuf) / *(int64_t *)pResInfo->interResultBuf; } else { // this is the secondary merge, only in the secondary merge, the input type is TSDB_DATA_TYPE_BINARY assert(pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_DOUBLE); - + SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf; - + if (pAvgInfo->num == 0) { // all data are NULL or empty table setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); return; } - + *(double *)pCtx->aOutputBuf = pAvgInfo->sum / pAvgInfo->num; } - + // cannot set the numOfIteratedElems again since it is set during previous iteration GET_RES_INFO(pCtx)->numOfRes = 1; doFinalizer(pCtx); @@ -928,22 +919,22 @@ static void avg_finalizer(SQLFunctionCtx *pCtx) { ///////////////////////////////////////////////////////////////////////////////////////////// static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, int32_t *notNullElems) { - if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus) && pCtx->preAggVals.isSet) { - // data in current data block are qualified to the query - *notNullElems = pCtx->size - pCtx->preAggVals.numOfNull; + // data in current data block are qualified to the query + if (usePreVal(pCtx)) { + *notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull; assert(*notNullElems >= 0); - + void * tval = NULL; int16_t index = 0; - + if (isMin) { - tval = &pCtx->preAggVals.min; - index = pCtx->preAggVals.minIndex; + tval = &pCtx->preAggVals.statis.min; + index = pCtx->preAggVals.statis.minIndex; } else { - tval = &pCtx->preAggVals.max; - index = pCtx->preAggVals.maxIndex; + tval = &pCtx->preAggVals.statis.max; + index = pCtx->preAggVals.statis.maxIndex; } - + /** * NOTE: work around the bug caused by invalid pre-calculated function. * Here the selectivity + ts will not return correct value. @@ -955,23 +946,23 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, } TSKEY key = pCtx->ptsList[index]; - + if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { int64_t val = GET_INT64_VAL(tval); if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { int8_t *data = (int8_t *)pOutput; - + UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { int16_t *data = (int16_t *)pOutput; - + UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { int32_t *data = (int32_t *)pOutput; #if defined(_DEBUG_VIEW) pTrace("max value updated according to pre-cal:%d", *data); #endif - + if ((*data < val) ^ isMin) { *data = val; for (int32_t i = 0; i < (pCtx)->tagInfo.numOfTagCols; ++i) { @@ -979,7 +970,7 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, if (__ctx->functionId == TSDB_FUNC_TS_DUMMY) { __ctx->tag = (tVariant){.i64Key = key, .nType = TSDB_DATA_TYPE_BIGINT}; } - + aAggs[TSDB_FUNC_TAG].xFunction(__ctx); } } @@ -990,21 +981,21 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { double *data = (double *)pOutput; double val = GET_DOUBLE_VAL(tval); - + UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { float *data = (float *)pOutput; double val = GET_DOUBLE_VAL(tval); - + UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key); } - + return; } - + void *p = GET_INPUT_CHAR(pCtx); *notNullElems = 0; - + if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { TYPED_LOOPCHECK_N(int8_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); @@ -1013,19 +1004,19 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { int32_t *pData = p; int32_t *retVal = (int32_t*) pOutput; - + for (int32_t i = 0; i < pCtx->size; ++i) { if (pCtx->hasNull && isNull((const char*)&pData[i], pCtx->inputType)) { continue; } - + if ((*retVal < pData[i]) ^ isMin) { *retVal = pData[i]; TSKEY k = pCtx->ptsList[i]; - + DO_UPDATE_TAG_COLUMNS(pCtx, k); } - + *notNullElems += 1; } #if defined(_DEBUG_VIEW) @@ -1045,9 +1036,9 @@ static bool min_func_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { return false; // not initialized since it has been initialized } - + GET_TRUE_DATA_TYPE(); - + switch (type) { case TSDB_DATA_TYPE_INT: *((int32_t *)pCtx->aOutputBuf) = INT32_MAX; @@ -1070,7 +1061,7 @@ static bool min_func_setup(SQLFunctionCtx *pCtx) { default: pError("illegal data type:%d in min/max query", pCtx->inputType); } - + return true; } @@ -1078,9 +1069,9 @@ static bool max_func_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { return false; // not initialized since it has been initialized } - + GET_TRUE_DATA_TYPE(); - + switch (type) { case TSDB_DATA_TYPE_INT: *((int32_t *)pCtx->aOutputBuf) = INT32_MIN; @@ -1103,7 +1094,7 @@ static bool max_func_setup(SQLFunctionCtx *pCtx) { default: pError("illegal data type:%d in min/max query", pCtx->inputType); } - + return true; } @@ -1113,13 +1104,13 @@ static bool max_func_setup(SQLFunctionCtx *pCtx) { static void min_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; minMax_function(pCtx, pCtx->aOutputBuf, 1, ¬NullElems); - + SET_VAL(pCtx, notNullElems, 1); - + if (notNullElems > 0) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; - + // set the flag for super table query if (pResInfo->superTableQ) { *(pCtx->aOutputBuf + pCtx->inputBytes) = DATA_SET_FLAG; @@ -1130,13 +1121,13 @@ static void min_function(SQLFunctionCtx *pCtx) { static void max_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; minMax_function(pCtx, pCtx->aOutputBuf, 0, ¬NullElems); - + SET_VAL(pCtx, notNullElems, 1); - + if (notNullElems > 0) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; - + // set the flag for super table query if (pResInfo->superTableQ) { *(pCtx->aOutputBuf + pCtx->inputBytes) = DATA_SET_FLAG; @@ -1146,18 +1137,18 @@ static void max_function(SQLFunctionCtx *pCtx) { static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *output, bool isMin) { int32_t notNullElems = 0; - + GET_TRUE_DATA_TYPE(); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); assert(pResInfo->superTableQ); - + for (int32_t i = 0; i < pCtx->size; ++i) { char *input = GET_INPUT_CHAR_INDEX(pCtx, i); if (input[bytes] != DATA_SET_FLAG) { continue; } - + switch (type) { case TSDB_DATA_TYPE_TINYINT: { int8_t v = GET_INT8_VAL(input); @@ -1173,12 +1164,12 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp int32_t v = GET_INT32_VAL(input); if ((*(int32_t *)output < v) ^ isMin) { *(int32_t *)output = v; - + for (int32_t i = 0; i < pCtx->tagInfo.numOfTagCols; ++i) { SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[i]; aAggs[TSDB_FUNC_TAG].xFunction(__ctx); } - + notNullElems++; } break; @@ -1202,15 +1193,15 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp break; } } - + return notNullElems; } static void min_func_merge(SQLFunctionCtx *pCtx) { int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->inputBytes, pCtx->aOutputBuf, 1); - + SET_VAL(pCtx, notNullElems, 1); - + if (notNullElems > 0) { // for super table query, SResultInfo is not used char *flag = pCtx->aOutputBuf + pCtx->inputBytes; *flag = DATA_SET_FLAG; @@ -1219,9 +1210,9 @@ static void min_func_merge(SQLFunctionCtx *pCtx) { static void min_func_second_merge(SQLFunctionCtx *pCtx) { int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->aOutputBuf, 1); - + SET_VAL(pCtx, notNullElems, 1); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); if (notNullElems > 0) { pResInfo->hasResult = DATA_SET_FLAG; @@ -1230,7 +1221,7 @@ static void min_func_second_merge(SQLFunctionCtx *pCtx) { static void max_func_merge(SQLFunctionCtx *pCtx) { int32_t numOfElems = minmax_merge_impl(pCtx, pCtx->inputBytes, pCtx->aOutputBuf, 0); - + SET_VAL(pCtx, numOfElems, 1); if (numOfElems > 0) { char *flag = pCtx->aOutputBuf + pCtx->inputBytes; @@ -1240,9 +1231,9 @@ static void max_func_merge(SQLFunctionCtx *pCtx) { static void max_func_second_merge(SQLFunctionCtx *pCtx) { int32_t numOfElem = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->aOutputBuf, 0); - + SET_VAL(pCtx, numOfElem, 1); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); if (numOfElem > 0) { pResInfo->hasResult = DATA_SET_FLAG; @@ -1252,40 +1243,40 @@ static void max_func_second_merge(SQLFunctionCtx *pCtx) { static void minMax_function_f(SQLFunctionCtx *pCtx, int32_t index, int32_t isMin) { char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); TSKEY key = pCtx->ptsList[index]; - + int32_t num = 0; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { int8_t *output = (int8_t *)pCtx->aOutputBuf; int8_t i = GET_INT8_VAL(pData); - + UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { int16_t *output = (int16_t*) pCtx->aOutputBuf; int16_t i = GET_INT16_VAL(pData); - + UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { int32_t *output = (int32_t*) pCtx->aOutputBuf; int32_t i = GET_INT32_VAL(pData); - + UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { int64_t *output = (int64_t*) pCtx->aOutputBuf; int64_t i = GET_INT64_VAL(pData); - + UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { float *output = (float*) pCtx->aOutputBuf; float i = GET_FLOAT_VAL(pData); - + UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { double *output = (double*) pCtx->aOutputBuf; double i = GET_DOUBLE_VAL(pData); - + UPDATE_DATA(pCtx, *output, i, num, isMin, key); } - + GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } @@ -1294,10 +1285,10 @@ static void max_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + SET_VAL(pCtx, 1, 1); minMax_function_f(pCtx, index, 0); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult == DATA_SET_FLAG) { char *flag = pCtx->aOutputBuf + pCtx->inputBytes; @@ -1310,10 +1301,10 @@ static void min_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + SET_VAL(pCtx, 1, 1); minMax_function_f(pCtx, index, 1); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult == DATA_SET_FLAG) { char *flag = pCtx->aOutputBuf + pCtx->inputBytes; @@ -1332,15 +1323,15 @@ static void min_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void stddev_function(SQLFunctionCtx *pCtx) { // the second stage to calculate standard deviation SStddevInfo *pStd = GET_RES_INFO(pCtx)->interResultBuf; - + if (pStd->stage == 0) { // the first stage is to calculate average value avg_function(pCtx); } else { double *retVal = &pStd->res; double avg = pStd->avg; - + void *pData = GET_INPUT_CHAR(pCtx); - + switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { for (int32_t i = 0; i < pCtx->size; ++i) { @@ -1374,7 +1365,7 @@ static void stddev_function(SQLFunctionCtx *pCtx) { default: pError("stddev function not support data type:%d", pCtx->inputType); } - + // TODO get the correct data SET_VAL(pCtx, 1, 1); } @@ -1384,18 +1375,18 @@ static void stddev_function_f(SQLFunctionCtx *pCtx, int32_t index) { // the second stage to calculate standard deviation SResultInfo *pResInfo = GET_RES_INFO(pCtx); SStddevInfo *pStd = pResInfo->interResultBuf; - + /* the first stage is to calculate average value */ if (pStd->stage == 0) { avg_function_f(pCtx, index); } else { double avg = pStd->avg; void * pData = GET_INPUT_CHAR_INDEX(pCtx, index); - + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { pStd->res += POW2(GET_INT32_VAL(pData) - avg); @@ -1424,7 +1415,7 @@ static void stddev_function_f(SQLFunctionCtx *pCtx, int32_t index) { default: pError("stddev function not support data type:%d", pCtx->inputType); } - + SET_VAL(pCtx, 1, 1); } } @@ -1436,7 +1427,7 @@ static void stddev_next_step(SQLFunctionCtx *pCtx) { */ SResultInfo *pResInfo = GET_RES_INFO(pCtx); SStddevInfo *pStd = pResInfo->interResultBuf; - + if (pStd->stage == 0) { /* * stddev is calculated in two stage: @@ -1446,12 +1437,12 @@ static void stddev_next_step(SQLFunctionCtx *pCtx) { */ pStd->stage++; avg_finalizer(pCtx); - + pResInfo->initialized = true; // set it initialized to avoid re-initialization // save average value into tmpBuf, for second stage scan SAvgInfo *pAvg = pResInfo->interResultBuf; - + pStd->avg = GET_DOUBLE_VAL(pCtx->aOutputBuf); assert((isnan(pAvg->sum) && pAvg->num == 0) || (pStd->num == pAvg->num && pStd->avg == pAvg->sum)); } else { @@ -1461,7 +1452,7 @@ static void stddev_next_step(SQLFunctionCtx *pCtx) { static void stddev_finalizer(SQLFunctionCtx *pCtx) { SStddevInfo *pStd = (SStddevInfo *)GET_RES_INFO(pCtx)->interResultBuf; - + if (pStd->num <= 0) { setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); } else { @@ -1478,42 +1469,42 @@ static bool first_last_function_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { return false; } - + // used to keep the timestamp for comparison pCtx->param[1].nType = 0; pCtx->param[1].i64Key = 0; - + return true; } // todo opt for null block static void first_function(SQLFunctionCtx *pCtx) { - if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus) || pCtx->order == TSQL_SO_DESC) { + if (pCtx->order == TSQL_SO_DESC) { return; } - + int32_t notNullElems = 0; - + // handle the null value for (int32_t i = 0; i < pCtx->size; ++i) { char *data = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } - + memcpy(pCtx->aOutputBuf, data, pCtx->inputBytes); - + TSKEY k = pCtx->ptsList[i]; DO_UPDATE_TAG_COLUMNS(pCtx, k); - + SResultInfo *pInfo = GET_RES_INFO(pCtx); pInfo->hasResult = DATA_SET_FLAG; pInfo->complete = true; - + notNullElems++; break; } - + SET_VAL(pCtx, notNullElems, 1); } @@ -1521,18 +1512,18 @@ static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->order == TSQL_SO_DESC) { return; } - + void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + SET_VAL(pCtx, 1, 1); memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); TSKEY ts = pCtx->ptsList[index]; DO_UPDATE_TAG_COLUMNS(pCtx, ts); - + SResultInfo *pInfo = GET_RES_INFO(pCtx); pInfo->hasResult = DATA_SET_FLAG; pInfo->complete = true; // get the first not-null data, completed @@ -1540,14 +1531,14 @@ static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void first_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { int64_t *timestamp = pCtx->ptsList; - + SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); - + if (pInfo->hasResult != DATA_SET_FLAG || timestamp[index] < pInfo->ts) { memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); pInfo->hasResult = DATA_SET_FLAG; pInfo->ts = timestamp[index]; - + DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts); } } @@ -1560,34 +1551,34 @@ static void first_dist_function(SQLFunctionCtx *pCtx) { if (pCtx->size == 0) { return; } - + /* * do not to check data in the following cases: * 1. data block that are not loaded * 2. scan data files in desc order */ - if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus) || pCtx->order == TSQL_SO_DESC) { + if (pCtx->order == TSQL_SO_DESC) { return; } - + int32_t notNullElems = 0; - + // find the first not null value for (int32_t i = 0; i < pCtx->size; ++i) { char *data = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } - + first_data_assign_impl(pCtx, data, i); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; - + notNullElems++; break; } - + SET_VAL(pCtx, notNullElems, 1); } @@ -1595,32 +1586,32 @@ static void first_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->size == 0) { return; } - + char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + if (pCtx->order == TSQL_SO_DESC) { return; } - + first_data_assign_impl(pCtx, pData, index); - + SET_VAL(pCtx, 1, 1); } static void first_dist_func_merge(SQLFunctionCtx *pCtx) { char *pData = GET_INPUT_CHAR(pCtx); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); assert(pCtx->size == 1 && pResInfo->superTableQ); - + SFirstLastInfo *pInput = (SFirstLastInfo *)(pData + pCtx->inputBytes); if (pInput->hasResult != DATA_SET_FLAG) { return; } - + SFirstLastInfo *pOutput = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); if (pOutput->hasResult != DATA_SET_FLAG || pInput->ts < pOutput->ts) { memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes + sizeof(SFirstLastInfo)); @@ -1630,22 +1621,22 @@ static void first_dist_func_merge(SQLFunctionCtx *pCtx) { static void first_dist_func_second_merge(SQLFunctionCtx *pCtx) { assert(pCtx->resultInfo->superTableQ); - + char * pData = GET_INPUT_CHAR(pCtx); SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes); if (pInput->hasResult != DATA_SET_FLAG) { return; } - + // The param[1] is used to keep the initial value of max ts value if (pCtx->param[1].nType != pCtx->outputType || pCtx->param[1].i64Key > pInput->ts) { memcpy(pCtx->aOutputBuf, pData, pCtx->outputBytes); pCtx->param[1].i64Key = pInput->ts; pCtx->param[1].nType = pCtx->outputType; - + DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); } - + SET_VAL(pCtx, 1, 1); GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } @@ -1659,31 +1650,31 @@ static void first_dist_func_second_merge(SQLFunctionCtx *pCtx) { * least one data in this block that is not null.(TODO opt for this case) */ static void last_function(SQLFunctionCtx *pCtx) { - if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus) || pCtx->order == TSQL_SO_ASC) { + if (pCtx->order == TSQL_SO_ASC) { return; } - + int32_t notNullElems = 0; - + for (int32_t i = pCtx->size - 1; i >= 0; --i) { char *data = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } - + memcpy(pCtx->aOutputBuf, data, pCtx->inputBytes); - + TSKEY ts = pCtx->ptsList[i]; DO_UPDATE_TAG_COLUMNS(pCtx, ts); - + SResultInfo *pInfo = GET_RES_INFO(pCtx); pInfo->hasResult = DATA_SET_FLAG; - + pInfo->complete = true; // set query completed on this column notNullElems++; break; } - + SET_VAL(pCtx, notNullElems, 1); } @@ -1691,18 +1682,18 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->order == TSQL_SO_ASC) { return; } - + void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + SET_VAL(pCtx, 1, 1); memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); TSKEY ts = pCtx->ptsList[index]; DO_UPDATE_TAG_COLUMNS(pCtx, ts); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; pResInfo->complete = true; // set query completed @@ -1710,18 +1701,18 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { int64_t *timestamp = pCtx->ptsList; - + SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); - + if (pInfo->hasResult != DATA_SET_FLAG || pInfo->ts < timestamp[index]) { #if defined(_DEBUG_VIEW) pTrace("assign index:%d, ts:%" PRId64 ", val:%d, ", index, timestamp[index], *(int32_t *)pData); #endif - + memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); pInfo->hasResult = DATA_SET_FLAG; pInfo->ts = timestamp[index]; - + DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts); } } @@ -1730,32 +1721,32 @@ static void last_dist_function(SQLFunctionCtx *pCtx) { if (pCtx->size == 0) { return; } - + /* * 1. for scan data in asc order, no need to check data * 2. for data blocks that are not loaded, no need to check data */ - if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus) || pCtx->order == TSQL_SO_ASC) { + if (pCtx->order == TSQL_SO_ASC) { return; } - + int32_t notNullElems = 0; - + for (int32_t i = pCtx->size - 1; i >= 0; --i) { char *data = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } - + last_data_assign_impl(pCtx, data, i); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; - + notNullElems++; break; } - + SET_VAL(pCtx, notNullElems, 1); } @@ -1763,12 +1754,12 @@ static void last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->size == 0) { return; } - + char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + /* * 1. for scan data in asc order, no need to check data * 2. for data blocks that are not loaded, no need to check data @@ -1776,28 +1767,28 @@ static void last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->order == TSQL_SO_ASC) { return; } - + last_data_assign_impl(pCtx, pData, index); - + SET_VAL(pCtx, 1, 1); } static void last_dist_func_merge(SQLFunctionCtx *pCtx) { char *pData = GET_INPUT_CHAR(pCtx); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); assert(pCtx->size == 1 && pResInfo->superTableQ); - + // the input data is null SFirstLastInfo *pInput = (SFirstLastInfo *)(pData + pCtx->inputBytes); if (pInput->hasResult != DATA_SET_FLAG) { return; } - + SFirstLastInfo *pOutput = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); if (pOutput->hasResult != DATA_SET_FLAG || pOutput->ts < pInput->ts) { memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes + sizeof(SFirstLastInfo)); - + DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); } } @@ -1809,12 +1800,12 @@ static void last_dist_func_merge(SQLFunctionCtx *pCtx) { */ static void last_dist_func_second_merge(SQLFunctionCtx *pCtx) { char *pData = GET_INPUT_CHAR(pCtx); - + SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes); if (pInput->hasResult != DATA_SET_FLAG) { return; } - + /* * param[1] used to keep the corresponding timestamp to decide if current result is * the true last result @@ -1823,10 +1814,10 @@ static void last_dist_func_second_merge(SQLFunctionCtx *pCtx) { memcpy(pCtx->aOutputBuf, pData, pCtx->outputBytes); pCtx->param[1].i64Key = pInput->ts; pCtx->param[1].nType = pCtx->outputType; - + DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); } - + SET_VAL(pCtx, 1, 1); GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } @@ -1837,25 +1828,25 @@ static void last_dist_func_second_merge(SQLFunctionCtx *pCtx) { */ static void last_row_function(SQLFunctionCtx *pCtx) { assert(pCtx->size == 1); - + char *pData = GET_INPUT_CHAR(pCtx); assignVal(pCtx->aOutputBuf, pData, pCtx->inputBytes, pCtx->inputType); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + SLastrowInfo *pInfo = (SLastrowInfo *)pResInfo->interResultBuf; pInfo->ts = pCtx->param[0].i64Key; pInfo->hasResult = DATA_SET_FLAG; - + // set the result to final result buffer if (pResInfo->superTableQ) { SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); pInfo1->ts = pCtx->param[0].i64Key; pInfo1->hasResult = DATA_SET_FLAG; - + DO_UPDATE_TAG_COLUMNS(pCtx, pInfo1->ts); } - + SET_VAL(pCtx, pCtx->size, 1); } @@ -1870,7 +1861,7 @@ static void last_row_finalizer(SQLFunctionCtx *pCtx) { } else { // do nothing } - + GET_RES_INFO(pCtx)->numOfRes = 1; doFinalizer(pCtx); } @@ -1881,7 +1872,7 @@ static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int6 dst->v.nType = type; dst->v.i64Key = *(int64_t *)val; dst->timestamp = tsKey; - + int32_t size = 0; if (stage == SECONDARY_STAGE_MERGE || stage == FIRST_STAGE_MERGE) { memcpy(dst->pTags, pTags, (size_t)pTagInfo->tagsLen); @@ -1909,20 +1900,20 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, SExtTagsInfo *pTagInfo, char *pTags, int16_t stage) { tVariant val = {0}; tVariantCreateFromBinary(&val, pData, tDataTypeDesc[type].nSize, type); - + tValuePair **pList = pInfo->res; assert(pList != NULL); if (pInfo->num < maxLen) { if (pInfo->num == 0 || ((type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) && - val.i64Key >= pList[pInfo->num - 1]->v.i64Key) || + val.i64Key >= pList[pInfo->num - 1]->v.i64Key) || ((type >= TSDB_DATA_TYPE_FLOAT && type <= TSDB_DATA_TYPE_DOUBLE) && - val.dKey >= pList[pInfo->num - 1]->v.dKey)) { + val.dKey >= pList[pInfo->num - 1]->v.dKey)) { valuePairAssign(pList[pInfo->num], type, (const char*)&val.i64Key, ts, pTags, pTagInfo, stage); } else { int32_t i = pInfo->num - 1; - + if (type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) { while (i >= 0 && pList[i]->v.i64Key > val.i64Key) { VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); @@ -1934,14 +1925,14 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, i -= 1; } } - + valuePairAssign(pList[i + 1], type, (const char*) &val.i64Key, ts, pTags, pTagInfo, stage); } - + pInfo->num++; } else { int32_t i = 0; - + if (((type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) && val.i64Key > pList[0]->v.i64Key) || ((type >= TSDB_DATA_TYPE_FLOAT && type <= TSDB_DATA_TYPE_DOUBLE) && val.dKey > pList[0]->v.dKey)) { // find the appropriate the slot position @@ -1956,7 +1947,7 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, i += 1; } } - + valuePairAssign(pList[i], type, (const char*) &val.i64Key, ts, pTags, pTagInfo, stage); } } @@ -1965,16 +1956,16 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, SExtTagsInfo *pTagInfo, char *pTags, int16_t stage) { tValuePair **pList = pInfo->res; - + tVariant val = {0}; tVariantCreateFromBinary(&val, pData, tDataTypeDesc[type].nSize, type); - + if (pInfo->num < maxLen) { if (pInfo->num == 0) { valuePairAssign(pList[pInfo->num], type, (const char*) &val.i64Key, ts, pTags, pTagInfo, stage); } else { int32_t i = pInfo->num - 1; - + if (type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) { while (i >= 0 && pList[i]->v.i64Key < val.i64Key) { VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); @@ -1986,14 +1977,14 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa i -= 1; } } - + valuePairAssign(pList[i + 1], type, (const char*)&val.i64Key, ts, pTags, pTagInfo, stage); } - + pInfo->num++; } else { int32_t i = 0; - + if (((type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) && val.i64Key < pList[0]->v.i64Key) || ((type >= TSDB_DATA_TYPE_FLOAT && type <= TSDB_DATA_TYPE_DOUBLE) && val.dKey < pList[0]->v.dKey)) { // find the appropriate the slot position @@ -2008,7 +1999,7 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa i += 1; } } - + valuePairAssign(pList[i], type, (const char*)&val.i64Key, ts, pTags, pTagInfo, stage); } } @@ -2017,7 +2008,7 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa static int32_t resAscComparFn(const void *pLeft, const void *pRight) { tValuePair *pLeftElem = *(tValuePair **)pLeft; tValuePair *pRightElem = *(tValuePair **)pRight; - + if (pLeftElem->timestamp == pRightElem->timestamp) { return 0; } else { @@ -2030,7 +2021,7 @@ static int32_t resDescComparFn(const void *pLeft, const void *pRight) { return - static int32_t resDataAscComparFn(const void *pLeft, const void *pRight) { tValuePair *pLeftElem = *(tValuePair **)pLeft; tValuePair *pRightElem = *(tValuePair **)pRight; - + int32_t type = pLeftElem->v.nType; if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { if (pLeftElem->v.dKey == pRightElem->v.dKey) { @@ -2052,12 +2043,12 @@ static int32_t resDataDescComparFn(const void *pLeft, const void *pRight) { retu static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); STopBotInfo *pRes = pResInfo->interResultBuf; - + tValuePair **tvp = pRes->res; int32_t step = QUERY_ASC_FORWARD_STEP; int32_t len = GET_RES_INFO(pCtx)->numOfRes; - + switch (type) { case TSDB_DATA_TYPE_INT: { int32_t *output = (int32_t *)pCtx->aOutputBuf; @@ -2106,20 +2097,20 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { return; } } - + // set the output timestamp of each record. TSKEY *output = pCtx->ptsOutputBuf; for (int32_t i = 0; i < len; ++i, output += step) { *output = tvp[i]->timestamp; } - + // set the corresponding tag data for each record // todo check malloc failure char **pData = calloc(pCtx->tagInfo.numOfTagCols, POINTER_BYTES); for (int32_t i = 0; i < pCtx->tagInfo.numOfTagCols; ++i) { pData[i] = pCtx->tagInfo.pTagCtxList[i]->aOutputBuf; } - + for (int32_t i = 0; i < len; ++i, output += step) { int16_t offset = 0; for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) { @@ -2128,22 +2119,22 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { pData[j] += pCtx->tagInfo.pTagCtxList[j]->outputBytes; } } - + tfree(pData); } bool top_bot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, char *minval, char *maxval) { STopBotInfo *pTopBotInfo = (STopBotInfo *)GET_RES_INFO(pCtx)->interResultBuf; - + int32_t numOfExistsRes = pTopBotInfo->num; - + // required number of results are not reached, continue load data block if (numOfExistsRes < pCtx->param[0].i64Key) { return true; } - + tValuePair *pRes = (tValuePair*) pTopBotInfo->res; - + if (functionId == TSDB_FUNC_TOP) { switch (pCtx->inputType) { case TSDB_DATA_TYPE_TINYINT: @@ -2191,7 +2182,7 @@ bool top_bot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, char *mi */ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + // only the first_stage_merge is directly written data into final output buffer if (pResInfo->superTableQ && pCtx->currentStage != SECONDARY_STAGE_MERGE) { return (STopBotInfo*) pCtx->aOutputBuf; @@ -2209,11 +2200,11 @@ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SQLFunctionCtx *pCtx) { char *tmp = (char *)pTopBotInfo + sizeof(STopBotInfo); pTopBotInfo->res = (tValuePair**) tmp; - + tmp += POINTER_BYTES * pCtx->param[0].i64Key; - + size_t size = sizeof(tValuePair) + pCtx->tagInfo.tagsLen; - + for (int32_t i = 0; i < pCtx->param[0].i64Key; ++i) { pTopBotInfo->res[i] = (tValuePair*) tmp; pTopBotInfo->res[i]->pTags = tmp + sizeof(tValuePair); @@ -2225,36 +2216,36 @@ static bool top_bottom_function_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { return false; } - + STopBotInfo *pInfo = getTopBotOutputInfo(pCtx); buildTopBotStruct(pInfo, pCtx); - + return true; } static void top_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - + STopBotInfo *pRes = getTopBotOutputInfo(pCtx); assert(pRes->num >= 0); - + for (int32_t i = 0; i < pCtx->size; ++i) { char *data = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } - + notNullElems++; do_top_function_add(pRes, pCtx->param[0].i64Key, data, pCtx->ptsList[i], pCtx->inputType, &pCtx->tagInfo, NULL, 0); } - + if (!pCtx->hasNull) { assert(pCtx->size == notNullElems); } - + // treat the result as only one result SET_VAL(pCtx, notNullElems, 1); - + if (notNullElems > 0) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; @@ -2266,34 +2257,34 @@ static void top_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + STopBotInfo *pRes = getTopBotOutputInfo(pCtx); assert(pRes->num >= 0); - + SET_VAL(pCtx, 1, 1); do_top_function_add(pRes, pCtx->param[0].i64Key, pData, pCtx->ptsList[index], pCtx->inputType, &pCtx->tagInfo, NULL, 0); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } static void top_func_merge(SQLFunctionCtx *pCtx) { char *input = GET_INPUT_CHAR(pCtx); - + STopBotInfo *pInput = (STopBotInfo *)input; if (pInput->num <= 0) { return; } - + // remmap the input buffer may cause the struct pointer invalid, so rebuild the STopBotInfo is necessary buildTopBotStruct(pInput, pCtx); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); assert(pResInfo->superTableQ && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1); - + STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - + for (int32_t i = 0; i < pInput->num; ++i) { do_top_function_add(pOutput, pCtx->param[0].i64Key, &pInput->res[i]->v.i64Key, pInput->res[i]->timestamp, pCtx->inputType, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); @@ -2302,20 +2293,20 @@ static void top_func_merge(SQLFunctionCtx *pCtx) { static void top_func_second_merge(SQLFunctionCtx *pCtx) { STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_CHAR(pCtx); - + // construct the input data struct from binary data buildTopBotStruct(pInput, pCtx); - + STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - + // the intermediate result is binary, we only use the output data type for (int32_t i = 0; i < pInput->num; ++i) { do_top_function_add(pOutput, pCtx->param[0].i64Key, &pInput->res[i]->v.i64Key, pInput->res[i]->timestamp, pCtx->outputType, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); } - + SET_VAL(pCtx, pInput->num, pOutput->num); - + if (pOutput->num > 0) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; @@ -2324,27 +2315,27 @@ static void top_func_second_merge(SQLFunctionCtx *pCtx) { static void bottom_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - + STopBotInfo *pRes = getTopBotOutputInfo(pCtx); - + for (int32_t i = 0; i < pCtx->size; ++i) { char *data = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } - + notNullElems++; do_bottom_function_add(pRes, pCtx->param[0].i64Key, data, pCtx->ptsList[i], pCtx->inputType, &pCtx->tagInfo, NULL, 0); } - + if (!pCtx->hasNull) { assert(pCtx->size == notNullElems); } - + // treat the result as only one result SET_VAL(pCtx, notNullElems, 1); - + if (notNullElems > 0) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; @@ -2356,32 +2347,32 @@ static void bottom_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + STopBotInfo *pRes = getTopBotOutputInfo(pCtx); SET_VAL(pCtx, 1, 1); do_bottom_function_add(pRes, pCtx->param[0].i64Key, pData, pCtx->ptsList[index], pCtx->inputType, &pCtx->tagInfo, NULL, 0); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } static void bottom_func_merge(SQLFunctionCtx *pCtx) { char *input = GET_INPUT_CHAR(pCtx); - + STopBotInfo *pInput = (STopBotInfo *)input; if (pInput->num <= 0) { return; } - + // remmap the input buffer may cause the struct pointer invalid, so rebuild the STopBotInfo is necessary buildTopBotStruct(pInput, pCtx); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); assert(pResInfo->superTableQ && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1); - + STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - + for (int32_t i = 0; i < pInput->num; ++i) { do_bottom_function_add(pOutput, pCtx->param[0].i64Key, &pInput->res[i]->v.i64Key, pInput->res[i]->timestamp, pCtx->inputType, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); @@ -2390,20 +2381,20 @@ static void bottom_func_merge(SQLFunctionCtx *pCtx) { static void bottom_func_second_merge(SQLFunctionCtx *pCtx) { STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_CHAR(pCtx); - + // construct the input data struct from binary data buildTopBotStruct(pInput, pCtx); - + STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - + // the intermediate result is binary, we only use the output data type for (int32_t i = 0; i < pInput->num; ++i) { do_bottom_function_add(pOutput, pCtx->param[0].i64Key, &pInput->res[i]->v.i64Key, pInput->res[i]->timestamp, pCtx->outputType, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); } - + SET_VAL(pCtx, pInput->num, pOutput->num); - + if (pOutput->num > 0) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; @@ -2412,17 +2403,17 @@ static void bottom_func_second_merge(SQLFunctionCtx *pCtx) { static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + // data in temporary list is less than the required number of results, not enough qualified number of results STopBotInfo *pRes = pResInfo->interResultBuf; if (pRes->num == 0) { // no result assert(pResInfo->hasResult != DATA_SET_FLAG); // TODO: } - + GET_RES_INFO(pCtx)->numOfRes = pRes->num; tValuePair **tvp = pRes->res; - + // user specify the order of output by sort the result according to timestamp if (pCtx->param[1].i64Key == PRIMARYKEY_TIMESTAMP_COL_INDEX) { __compar_fn_t comparator = (pCtx->param[2].i64Key == TSQL_SO_ASC) ? resAscComparFn : resDescComparFn; @@ -2431,7 +2422,7 @@ static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { __compar_fn_t comparator = (pCtx->param[2].i64Key == TSQL_SO_ASC) ? resDataAscComparFn : resDataDescComparFn; qsort(tvp, pResInfo->numOfRes, POINTER_BYTES, comparator); } - + GET_TRUE_DATA_TYPE(); copyTopBotRes(pCtx, type); @@ -2442,42 +2433,42 @@ static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { static bool percentile_function_setup(SQLFunctionCtx *pCtx) { const int32_t MAX_AVAILABLE_BUFFER_SIZE = 1 << 20; // 1MB const int32_t NUMOFCOLS = 1; - + if (!function_setup(pCtx)) { return false; } - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); SSchema field[1] = {{pCtx->inputType, "dummyCol", 0, pCtx->inputBytes}}; - + SColumnModel *pModel = createColumnModel(field, 1, 1000); int32_t orderIdx = 0; - + // tOrderDesc object tOrderDescriptor *pDesc = tOrderDesCreate(&orderIdx, NUMOFCOLS, pModel, TSQL_SO_DESC); - + ((SPercentileInfo *)(pResInfo->interResultBuf))->pMemBucket = tMemBucketCreate(1024, MAX_AVAILABLE_BUFFER_SIZE, pCtx->inputBytes, pCtx->inputType, pDesc); - + return true; } static void percentile_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - + SResultInfo * pResInfo = GET_RES_INFO(pCtx); SPercentileInfo *pInfo = pResInfo->interResultBuf; - + for (int32_t i = 0; i < pCtx->size; ++i) { char *data = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } - + notNullElems += 1; tMemBucketPut(pInfo->pMemBucket, data, 1); } - + SET_VAL(pCtx, notNullElems, 1); pResInfo->hasResult = DATA_SET_FLAG; } @@ -2487,28 +2478,28 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + SPercentileInfo *pInfo = (SPercentileInfo *)pResInfo->interResultBuf; tMemBucketPut(pInfo->pMemBucket, pData, 1); - + SET_VAL(pCtx, 1, 1); pResInfo->hasResult = DATA_SET_FLAG; } static void percentile_finalizer(SQLFunctionCtx *pCtx) { double v = pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i64Key : pCtx->param[0].dKey; - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); tMemBucket * pMemBucket = ((SPercentileInfo *)pResInfo->interResultBuf)->pMemBucket; - + if (pMemBucket->numOfElems > 0) { // check for null *(double *)pCtx->aOutputBuf = getPercentile(pMemBucket, v); } else { setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); } - + tOrderDescDestroy(pMemBucket->pOrderDesc); tMemBucketDestroy(pMemBucket); @@ -2518,7 +2509,7 @@ static void percentile_finalizer(SQLFunctionCtx *pCtx) { ////////////////////////////////////////////////////////////////////////////////// static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + if (pResInfo->superTableQ && pCtx->currentStage != SECONDARY_STAGE_MERGE) { return (SAPercentileInfo*) pCtx->aOutputBuf; } else { @@ -2530,9 +2521,9 @@ static bool apercentile_function_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { return false; } - + SAPercentileInfo *pInfo = getAPerctInfo(pCtx); - + char *tmp = (char *)pInfo + sizeof(SAPercentileInfo); pInfo->pHisto = tHistogramCreateFrom(tmp, MAX_HISTOGRAM_BIN); return true; @@ -2540,19 +2531,19 @@ static bool apercentile_function_setup(SQLFunctionCtx *pCtx) { static void apercentile_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - + SResultInfo * pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo *pInfo = getAPerctInfo(pCtx); - + for (int32_t i = 0; i < pCtx->size; ++i) { char *data = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } - + notNullElems += 1; double v = 0; - + switch (pCtx->inputType) { case TSDB_DATA_TYPE_TINYINT: v = GET_INT8_VAL(data); @@ -2573,16 +2564,16 @@ static void apercentile_function(SQLFunctionCtx *pCtx) { v = GET_INT32_VAL(data); break; } - + tHistogramAdd(&pInfo->pHisto, v); } - + if (!pCtx->hasNull) { assert(pCtx->size == notNullElems); } - + SET_VAL(pCtx, notNullElems, 1); - + if (notNullElems > 0) { pResInfo->hasResult = DATA_SET_FLAG; } @@ -2593,10 +2584,10 @@ static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + SResultInfo * pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo *pInfo = getAPerctInfo(pCtx); // pResInfo->interResultBuf; - + double v = 0; switch (pCtx->inputType) { case TSDB_DATA_TYPE_TINYINT: @@ -2618,9 +2609,9 @@ static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { v = GET_INT32_VAL(pData); break; } - + tHistogramAdd(&pInfo->pHisto, v); - + SET_VAL(pCtx, 1, 1); pResInfo->hasResult = DATA_SET_FLAG; } @@ -2628,62 +2619,62 @@ static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void apercentile_func_merge(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); assert(pResInfo->superTableQ); - + SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_CHAR(pCtx); - + pInput->pHisto = (SHistogramInfo*) ((char *)pInput + sizeof(SAPercentileInfo)); pInput->pHisto->elems = (SHistBin*) ((char *)pInput->pHisto + sizeof(SHistogramInfo)); - + if (pInput->pHisto->numOfElems <= 0) { return; } - + size_t size = sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1); - + SAPercentileInfo *pOutput = getAPerctInfo(pCtx); //(SAPercentileInfo *)pCtx->aOutputBuf; SHistogramInfo * pHisto = pOutput->pHisto; - + if (pHisto->numOfElems <= 0) { memcpy(pHisto, pInput->pHisto, size); pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); } else { pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - + SHistogramInfo *pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN); memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN); pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - + tHistogramDestroy(&pRes); } - + SET_VAL(pCtx, 1, 1); pResInfo->hasResult = DATA_SET_FLAG; } static void apercentile_func_second_merge(SQLFunctionCtx *pCtx) { SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_CHAR(pCtx); - + pInput->pHisto = (SHistogramInfo*) ((char *)pInput + sizeof(SAPercentileInfo)); pInput->pHisto->elems = (SHistBin*) ((char *)pInput->pHisto + sizeof(SHistogramInfo)); - + if (pInput->pHisto->numOfElems <= 0) { return; } - + SAPercentileInfo *pOutput = getAPerctInfo(pCtx); SHistogramInfo * pHisto = pOutput->pHisto; - + if (pHisto->numOfElems <= 0) { memcpy(pHisto, pInput->pHisto, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1)); pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); } else { pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - + SHistogramInfo *pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN); tHistogramDestroy(&pOutput->pHisto); pOutput->pHisto = pRes; } - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; SET_VAL(pCtx, 1, 1); @@ -2691,17 +2682,17 @@ static void apercentile_func_second_merge(SQLFunctionCtx *pCtx) { static void apercentile_finalizer(SQLFunctionCtx *pCtx) { double v = (pCtx->param[0].nType == TSDB_DATA_TYPE_INT) ? pCtx->param[0].i64Key : pCtx->param[0].dKey; - + SResultInfo * pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo *pOutput = pResInfo->interResultBuf; - + if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { if (pResInfo->hasResult == DATA_SET_FLAG) { // check for null assert(pOutput->pHisto->numOfElems > 0); - + double ratio[] = {v}; double *res = tHistogramUniform(pOutput->pHisto, ratio, 1); - + memcpy(pCtx->aOutputBuf, res, sizeof(double)); free(res); } else { @@ -2711,7 +2702,7 @@ static void apercentile_finalizer(SQLFunctionCtx *pCtx) { } else { if (pOutput->pHisto->numOfElems > 0) { double ratio[] = {v}; - + double *res = tHistogramUniform(pOutput->pHisto, ratio, 1); memcpy(pCtx->aOutputBuf, res, sizeof(double)); free(res); @@ -2729,10 +2720,10 @@ static bool leastsquares_function_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { return false; } - + SResultInfo * pResInfo = GET_RES_INFO(pCtx); SLeastsquareInfo *pInfo = pResInfo->interResultBuf; - + // 2*3 matrix pInfo->startVal = pCtx->param[0].dKey; return true; @@ -2759,12 +2750,12 @@ static bool leastsquares_function_setup(SQLFunctionCtx *pCtx) { static void leastsquares_function(SQLFunctionCtx *pCtx) { SResultInfo * pResInfo = GET_RES_INFO(pCtx); SLeastsquareInfo *pInfo = pResInfo->interResultBuf; - + double(*param)[3] = pInfo->mat; double x = pInfo->startVal; - + void *pData = GET_INPUT_CHAR(pCtx); - + int32_t numOfElem = 0; switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { @@ -2774,12 +2765,12 @@ static void leastsquares_function(SQLFunctionCtx *pCtx) { if (pCtx->hasNull && isNull((const char*) p, pCtx->inputType)) { continue; } - + param[0][0] += x * x; param[0][1] += x; param[0][2] += x * p[i]; param[1][2] += p[i]; - + x += pCtx->param[1].dKey; numOfElem++; } @@ -2811,14 +2802,14 @@ static void leastsquares_function(SQLFunctionCtx *pCtx) { break; }; } - + pInfo->startVal = x; pInfo->num += numOfElem; - + if (pInfo->num > 0) { pResInfo->hasResult = DATA_SET_FLAG; } - + SET_VAL(pCtx, numOfElem, 1); } @@ -2827,12 +2818,12 @@ static void leastsquares_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + SResultInfo * pResInfo = GET_RES_INFO(pCtx); SLeastsquareInfo *pInfo = pResInfo->interResultBuf; - + double(*param)[3] = pInfo->mat; - + switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { int32_t *p = pData; @@ -2867,10 +2858,10 @@ static void leastsquares_function_f(SQLFunctionCtx *pCtx, int32_t index) { default: pError("error data type in leastsquare function:%d", pCtx->inputType); }; - + SET_VAL(pCtx, 1, 1); pInfo->num += 1; - + if (pInfo->num > 0) { pResInfo->hasResult = DATA_SET_FLAG; } @@ -2880,26 +2871,26 @@ static void leastsquares_finalizer(SQLFunctionCtx *pCtx) { // no data in query SResultInfo * pResInfo = GET_RES_INFO(pCtx); SLeastsquareInfo *pInfo = pResInfo->interResultBuf; - + if (pInfo->num == 0) { setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); return; } - + double(*param)[3] = pInfo->mat; - + param[1][1] = pInfo->num; param[1][0] = param[0][1]; - + param[0][0] -= param[1][0] * (param[0][1] / param[1][1]); param[0][2] -= param[1][2] * (param[0][1] / param[1][1]); param[0][1] = 0; param[1][2] -= param[0][2] * (param[1][0] / param[0][0]); param[1][0] = 0; param[0][2] /= param[0][0]; - + param[1][2] /= param[1][1]; - + sprintf(pCtx->aOutputBuf, "(%lf, %lf)", param[0][2], param[1][2]); doFinalizer(pCtx); } @@ -2908,7 +2899,7 @@ static void date_col_output_function(SQLFunctionCtx *pCtx) { if (pCtx->scanFlag == SUPPLEMENTARY_SCAN) { return; } - + SET_VAL(pCtx, pCtx->size, 1); *(int64_t *)(pCtx->aOutputBuf) = pCtx->nStartQueryTimestamp; } @@ -2919,32 +2910,32 @@ static FORCE_INLINE void date_col_output_function_f(SQLFunctionCtx *pCtx, int32_ static void col_project_function(SQLFunctionCtx *pCtx) { INC_INIT_VAL(pCtx, pCtx->size); - + char *pData = GET_INPUT_CHAR(pCtx); if (pCtx->order == TSQL_SO_ASC) { memcpy(pCtx->aOutputBuf, pData, (size_t)pCtx->size * pCtx->inputBytes); } else { for(int32_t i = 0; i < pCtx->size; ++i) { memcpy(pCtx->aOutputBuf + (pCtx->size - 1 - i) * pCtx->inputBytes, pData + i * pCtx->inputBytes, - pCtx->inputBytes); + pCtx->inputBytes); } } - + pCtx->aOutputBuf += pCtx->size * pCtx->outputBytes; } static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + // only one output if (pCtx->param[0].i64Key == 1 && pResInfo->numOfRes >= 1) { return; } - + INC_INIT_VAL(pCtx, 1); char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); - + pCtx->aOutputBuf += pCtx->inputBytes/* * GET_FORWARD_DIRECTION_FACTOR(pCtx->order)*/; } @@ -2955,9 +2946,9 @@ static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { */ static void tag_project_function(SQLFunctionCtx *pCtx) { INC_INIT_VAL(pCtx, pCtx->size); - + assert(pCtx->inputBytes == pCtx->outputBytes); - + for (int32_t i = 0; i < pCtx->size; ++i) { tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->outputType); pCtx->aOutputBuf += pCtx->outputBytes; @@ -2989,7 +2980,7 @@ static void tag_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void copy_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, pCtx->size, 1); - + char *pData = GET_INPUT_CHAR(pCtx); assignVal(pCtx->aOutputBuf, pData, pCtx->inputBytes, pCtx->inputType); } @@ -3002,7 +2993,7 @@ static bool diff_function_setup(SQLFunctionCtx *pCtx) { if (function_setup(pCtx)) { return false; } - + // diff function require the value is set to -1 pCtx->param[1].nType = INITIAL_VALUE_NOT_ASSIGNED; return false; @@ -3012,41 +3003,41 @@ static bool diff_function_setup(SQLFunctionCtx *pCtx) { static void diff_function(SQLFunctionCtx *pCtx) { void *data = GET_INPUT_CHAR(pCtx); bool isFirstBlock = (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED); - + int32_t notNullElems = 0; - + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); int32_t i = (pCtx->order == TSQL_SO_ASC) ? 0 : pCtx->size - 1; TSKEY * pTimestamp = pCtx->ptsOutputBuf; - + switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { int32_t *pData = (int32_t *)data; int32_t *pOutput = (int32_t *)pCtx->aOutputBuf; - + for (; i < pCtx->size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { continue; } - + if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet pCtx->param[1].i64Key = pData[i]; pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSQL_SO_ASC) || (i == pCtx->size - 1 && pCtx->order == TSQL_SO_DESC)) { *pOutput = pData[i] - pCtx->param[1].i64Key; *pTimestamp = pCtx->ptsList[i]; - + pOutput += 1; pTimestamp += 1; } else { *pOutput = pData[i] - pData[i - step]; *pTimestamp = pCtx->ptsList[i]; - + pOutput += 1; pTimestamp += 1; } - + pCtx->param[1].i64Key = pData[i]; pCtx->param[1].nType = pCtx->inputType; notNullElems++; @@ -3056,29 +3047,29 @@ static void diff_function(SQLFunctionCtx *pCtx) { case TSDB_DATA_TYPE_BIGINT: { int64_t *pData = (int64_t *)data; int64_t *pOutput = (int64_t *)pCtx->aOutputBuf; - + for (; i < pCtx->size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { continue; } - + if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet pCtx->param[1].i64Key = pData[i]; pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSQL_SO_ASC) || (i == pCtx->size - 1 && pCtx->order == TSQL_SO_DESC)) { *pOutput = pData[i] - pCtx->param[1].i64Key; *pTimestamp = pCtx->ptsList[i]; - + pOutput += 1; pTimestamp += 1; } else { *pOutput = pData[i] - pData[i - step]; *pTimestamp = pCtx->ptsList[i]; - + pOutput += 1; pTimestamp += 1; } - + pCtx->param[1].i64Key = pData[i]; pCtx->param[1].nType = pCtx->inputType; notNullElems++; @@ -3088,12 +3079,12 @@ static void diff_function(SQLFunctionCtx *pCtx) { case TSDB_DATA_TYPE_DOUBLE: { double *pData = (double *)data; double *pOutput = (double *)pCtx->aOutputBuf; - + for (; i < pCtx->size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { continue; } - + if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet pCtx->param[1].dKey = pData[i]; pCtx->param[1].nType = pCtx->inputType; @@ -3108,7 +3099,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { pOutput += 1; pTimestamp += 1; } - + pCtx->param[1].dKey = pData[i]; pCtx->param[1].nType = pCtx->inputType; notNullElems++; @@ -3118,12 +3109,12 @@ static void diff_function(SQLFunctionCtx *pCtx) { case TSDB_DATA_TYPE_FLOAT: { float *pData = (float *)data; float *pOutput = (float *)pCtx->aOutputBuf; - + for (; i < pCtx->size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { continue; } - + if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet pCtx->param[1].dKey = pData[i]; pCtx->param[1].nType = pCtx->inputType; @@ -3140,7 +3131,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { pOutput += 1; pTimestamp += 1; } - + // keep the last value, the remain may be all null pCtx->param[1].dKey = pData[i]; pCtx->param[1].nType = pCtx->inputType; @@ -3151,12 +3142,12 @@ static void diff_function(SQLFunctionCtx *pCtx) { case TSDB_DATA_TYPE_SMALLINT: { int16_t *pData = (int16_t *)data; int16_t *pOutput = (int16_t *)pCtx->aOutputBuf; - + for (; i < pCtx->size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { continue; } - + if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet pCtx->param[1].i64Key = pData[i]; pCtx->param[1].nType = pCtx->inputType; @@ -3172,7 +3163,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { pOutput += 1; pTimestamp += 1; } - + pCtx->param[1].i64Key = pData[i]; pCtx->param[1].nType = pCtx->inputType; notNullElems++; @@ -3182,12 +3173,12 @@ static void diff_function(SQLFunctionCtx *pCtx) { case TSDB_DATA_TYPE_TINYINT: { int8_t *pData = (int8_t *)data; int8_t *pOutput = (int8_t *)pCtx->aOutputBuf; - + for (; i < pCtx->size && i >= 0; i += step) { if (pCtx->hasNull && isNull((char *)&pData[i], pCtx->inputType)) { continue; } - + if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet pCtx->param[1].i64Key = pData[i]; pCtx->param[1].nType = pCtx->inputType; @@ -3204,7 +3195,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { pOutput += 1; pTimestamp += 1; } - + pCtx->param[1].i64Key = pData[i]; pCtx->param[1].nType = pCtx->inputType; notNullElems++; @@ -3214,7 +3205,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { default: pError("error input type"); } - + // initial value is not set yet if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED || notNullElems <= 0) { /* @@ -3224,9 +3215,9 @@ static void diff_function(SQLFunctionCtx *pCtx) { assert(pCtx->hasNull); } else { int32_t forwardStep = (isFirstBlock) ? notNullElems - 1 : notNullElems; - + GET_RES_INFO(pCtx)->numOfRes += forwardStep; - + pCtx->aOutputBuf += forwardStep * pCtx->outputBytes; pCtx->ptsOutputBuf = (char*)pCtx->ptsOutputBuf + forwardStep * TSDB_KEYSIZE; } @@ -3249,14 +3240,14 @@ static void diff_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + // the output start from the second source element if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is set GET_RES_INFO(pCtx)->numOfRes += 1; } - + int32_t step = 1/*GET_FORWARD_DIRECTION_FACTOR(pCtx->order)*/; - + switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet @@ -3292,7 +3283,7 @@ static void diff_function_f(SQLFunctionCtx *pCtx, int32_t index) { default: pError("error input type"); } - + if (GET_RES_INFO(pCtx)->numOfRes > 0) { pCtx->aOutputBuf += pCtx->outputBytes * step; pCtx->ptsOutputBuf = (char *)pCtx->ptsOutputBuf + TSDB_KEYSIZE * step; @@ -3301,17 +3292,17 @@ static void diff_function_f(SQLFunctionCtx *pCtx, int32_t index) { char *arithmetic_callback_function(void *param, char *name, int32_t colId) { SArithmeticSupport *pSupport = (SArithmeticSupport *)param; - + SSqlFunctionExpr *pExpr = pSupport->pExpr; int32_t colIndexInBuf = -1; - - for (int32_t i = 0; i < pExpr->pBinExprInfo.numOfCols; ++i) { - if (colId == pExpr->pBinExprInfo.pReqColumns[i].colId) { - colIndexInBuf = pExpr->pBinExprInfo.pReqColumns[i].colIdxInBuf; + + for (int32_t i = 0; i < pExpr->binExprInfo.numOfCols; ++i) { + if (colId == pExpr->binExprInfo.pReqColumns[i].colId) { + colIndexInBuf = pExpr->binExprInfo.pReqColumns[i].colIdxInBuf; break; } } - + assert(colIndexInBuf >= 0 && colId >= 0); return pSupport->data[colIndexInBuf] + pSupport->offset * pSupport->elemSize[colIndexInBuf]; } @@ -3319,10 +3310,10 @@ char *arithmetic_callback_function(void *param, char *name, int32_t colId) { static void arithmetic_function(SQLFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->numOfRes += pCtx->size; SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; - - tSQLBinaryExprCalcTraverse(sas->pExpr->pBinExprInfo.pBinExpr, pCtx->size, pCtx->aOutputBuf, sas, pCtx->order, + + tSQLBinaryExprCalcTraverse(sas->pExpr->binExprInfo.pBinExpr, pCtx->size, pCtx->aOutputBuf, sas, pCtx->order, arithmetic_callback_function); - + pCtx->aOutputBuf += pCtx->outputBytes * pCtx->size; pCtx->param[1].pz = NULL; } @@ -3330,11 +3321,11 @@ static void arithmetic_function(SQLFunctionCtx *pCtx) { static void arithmetic_function_f(SQLFunctionCtx *pCtx, int32_t index) { INC_INIT_VAL(pCtx, 1); SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; - + sas->offset = index; - tSQLBinaryExprCalcTraverse(sas->pExpr->pBinExprInfo.pBinExpr, 1, pCtx->aOutputBuf, sas, pCtx->order, + tSQLBinaryExprCalcTraverse(sas->pExpr->binExprInfo.pBinExpr, 1, pCtx->aOutputBuf, sas, pCtx->order, arithmetic_callback_function); - + pCtx->aOutputBuf += pCtx->outputBytes/* * GET_FORWARD_DIRECTION_FACTOR(pCtx->order)*/; } @@ -3360,9 +3351,9 @@ static bool spread_function_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { return false; } - + SSpreadInfo *pInfo = GET_RES_INFO(pCtx)->interResultBuf; - + // this is the server-side setup function in client-side, the secondary merge do not need this procedure if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { pCtx->param[0].dKey = DBL_MAX; @@ -3371,60 +3362,56 @@ static bool spread_function_setup(SQLFunctionCtx *pCtx) { pInfo->min = DBL_MAX; pInfo->max = -DBL_MAX; } - + return true; } static void spread_function(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); SSpreadInfo *pInfo = pResInfo->interResultBuf; - + int32_t numOfElems = pCtx->size; - + // column missing cause the hasNull to be true - if (!IS_DATA_BLOCK_LOADED(pCtx->blockStatus)) { - if (pCtx->preAggVals.isSet) { - numOfElems = pCtx->size - pCtx->preAggVals.numOfNull; - - // all data are null in current data block, ignore current data block - if (numOfElems == 0) { - goto _spread_over; + if (usePreVal(pCtx)) { + numOfElems = pCtx->size - pCtx->preAggVals.statis.numOfNull; + + // all data are null in current data block, ignore current data block + if (numOfElems == 0) { + goto _spread_over; + } + + if ((pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) || + (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)) { + if (pInfo->min > pCtx->preAggVals.statis.min) { + pInfo->min = pCtx->preAggVals.statis.min; } - - if ((pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) || - (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)) { - if (pInfo->min > pCtx->preAggVals.min) { - pInfo->min = pCtx->preAggVals.min; - } - - if (pInfo->max < pCtx->preAggVals.max) { - pInfo->max = pCtx->preAggVals.max; - } - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - if (pInfo->min > GET_DOUBLE_VAL(&(pCtx->preAggVals.min))) { - pInfo->min = GET_DOUBLE_VAL(&(pCtx->preAggVals.min)); - } - - if (pInfo->max < GET_DOUBLE_VAL(&(pCtx->preAggVals.max))) { - pInfo->max = GET_DOUBLE_VAL(&(pCtx->preAggVals.max)); - } + + if (pInfo->max < pCtx->preAggVals.statis.max) { + pInfo->max = pCtx->preAggVals.statis.max; } - } else { - if (pInfo->min > pCtx->param[1].dKey) { - pInfo->min = pCtx->param[1].dKey; + } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { + if (pInfo->min > GET_DOUBLE_VAL(&(pCtx->preAggVals.statis.min))) { + pInfo->min = GET_DOUBLE_VAL(&(pCtx->preAggVals.statis.min)); } - - if (pInfo->max < pCtx->param[2].dKey) { - pInfo->max = pCtx->param[2].dKey; + + if (pInfo->max < GET_DOUBLE_VAL(&(pCtx->preAggVals.statis.max))) { + pInfo->max = GET_DOUBLE_VAL(&(pCtx->preAggVals.statis.max)); } } - - goto _spread_over; + } else { + if (pInfo->min > pCtx->param[1].dKey) { + pInfo->min = pCtx->param[1].dKey; + } + + if (pInfo->max < pCtx->param[2].dKey) { + pInfo->max = pCtx->param[2].dKey; + } } - + void *pData = GET_INPUT_CHAR(pCtx); numOfElems = 0; - + if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int8_t, pCtx->inputType, numOfElems); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { @@ -3438,19 +3425,19 @@ static void spread_function(SQLFunctionCtx *pCtx) { } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, float, pCtx->inputType, numOfElems); } - + if (!pCtx->hasNull) { assert(pCtx->size == numOfElems); } - -_spread_over: + + _spread_over: SET_VAL(pCtx, numOfElems, 1); - + if (numOfElems > 0) { pResInfo->hasResult = DATA_SET_FLAG; pInfo->hasResult = DATA_SET_FLAG; } - + // keep the data into the final output buffer for super table query since this execution may be the last one if (pResInfo->superTableQ) { memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SSpreadInfo)); @@ -3462,12 +3449,12 @@ static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + SET_VAL(pCtx, 1, 1); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); SSpreadInfo *pInfo = pResInfo->interResultBuf; - + double val = 0.0; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { val = GET_INT8_VAL(pData); @@ -3482,19 +3469,19 @@ static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) { } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { val = GET_FLOAT_VAL(pData); } - + // keep the result data in output buffer, not in the intermediate buffer if (val > pInfo->max) { pInfo->max = val; } - + if (val < pInfo->min) { pInfo->min = val; } - + pResInfo->hasResult = DATA_SET_FLAG; pInfo->hasResult = DATA_SET_FLAG; - + if (pResInfo->superTableQ) { memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SSpreadInfo)); } @@ -3503,30 +3490,30 @@ static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) { void spread_func_merge(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); assert(pResInfo->superTableQ); - + SSpreadInfo *pResData = pResInfo->interResultBuf; - + int32_t notNullElems = 0; for (int32_t i = 0; i < pCtx->size; ++i) { SSpreadInfo *input = (SSpreadInfo *)GET_INPUT_CHAR_INDEX(pCtx, i); - + /* no assign tag, the value is null */ if (input->hasResult != DATA_SET_FLAG) { continue; } - + if (pResData->min > input->min) { pResData->min = input->min; } - + if (pResData->max < input->max) { pResData->max = input->max; } - + pResData->hasResult = DATA_SET_FLAG; notNullElems++; } - + if (notNullElems > 0) { memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SSpreadInfo)); pResInfo->hasResult = DATA_SET_FLAG; @@ -3542,15 +3529,15 @@ void spread_func_sec_merge(SQLFunctionCtx *pCtx) { if (pData->hasResult != DATA_SET_FLAG) { return; } - + if (pCtx->param[0].dKey > pData->min) { pCtx->param[0].dKey = pData->min; } - + if (pCtx->param[3].dKey < pData->max) { pCtx->param[3].dKey = pData->max; } - + GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } @@ -3560,26 +3547,26 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { * the type of intermediate data is binary */ SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - + if (pResInfo->hasResult != DATA_SET_FLAG) { setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); return; } - + *(double *)pCtx->aOutputBuf = pCtx->param[3].dKey - pCtx->param[0].dKey; } else { assert((pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_DOUBLE) || - (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)); - + (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)); + SSpreadInfo *pInfo = GET_RES_INFO(pCtx)->interResultBuf; if (pInfo->hasResult != DATA_SET_FLAG) { setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); return; } - + *(double *)pCtx->aOutputBuf = pInfo->max - pInfo->min; } @@ -3599,33 +3586,33 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { */ int patternMatch(const char *patterStr, const char *str, size_t size, const SPatternCompareInfo *pInfo) { char c, c1; - + int32_t i = 0; int32_t j = 0; - + while ((c = patterStr[i++]) != 0) { if (c == pInfo->matchAll) { /* Match "*" */ - + while ((c = patterStr[i++]) == pInfo->matchAll || c == pInfo->matchOne) { if (c == pInfo->matchOne && (j > size || str[j++] == 0)) { // empty string, return not match return TSDB_PATTERN_NOWILDCARDMATCH; } } - + if (c == 0) { return TSDB_PATTERN_MATCH; /* "*" at the end of the pattern matches */ } - + char next[3] = {toupper(c), tolower(c), 0}; while (1) { size_t n = strcspn(str, next); str += n; - + if (str[0] == 0 || (n >= size - 1)) { break; } - + int32_t ret = patternMatch(&patterStr[i], ++str, size - n - 1, pInfo); if (ret != TSDB_PATTERN_NOMATCH) { return ret; @@ -3633,18 +3620,18 @@ int patternMatch(const char *patterStr, const char *str, size_t size, const SPat } return TSDB_PATTERN_NOWILDCARDMATCH; } - + c1 = str[j++]; - + if (j <= size) { if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) { continue; } } - + return TSDB_PATTERN_NOMATCH; } - + return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; } @@ -3652,13 +3639,13 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c wchar_t c, c1; wchar_t matchOne = L'_'; // "_" wchar_t matchAll = L'%'; // "%" - + int32_t i = 0; int32_t j = 0; - + while ((c = patterStr[i++]) != 0) { if (c == matchAll) { /* Match "%" */ - + while ((c = patterStr[i++]) == matchAll || c == matchOne) { if (c == matchOne && (j > size || str[j++] == 0)) { return TSDB_PATTERN_NOWILDCARDMATCH; @@ -3667,38 +3654,38 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c if (c == 0) { return TSDB_PATTERN_MATCH; } - + wchar_t accept[3] = {towupper(c), towlower(c), 0}; while (1) { size_t n = wcsspn(str, accept); - + str += n; if (str[0] == 0 || (n >= size - 1)) { break; } - + str++; - + int32_t ret = WCSPatternMatch(&patterStr[i], str, wcslen(str), pInfo); if (ret != TSDB_PATTERN_NOMATCH) { return ret; } } - + return TSDB_PATTERN_NOWILDCARDMATCH; } - + c1 = str[j++]; - + if (j <= size) { if (c == c1 || towlower(c) == towlower(c1) || (c == matchOne && c1 != 0)) { continue; } } - + return TSDB_PATTERN_NOMATCH; } - + return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; } @@ -3708,29 +3695,29 @@ static void getStatics_i8(int64_t *primaryKey, int32_t type, int8_t *data, int32 *max = INT64_MIN; *minIndex = 0; *maxIndex = 0; - + assert(numOfRow <= INT16_MAX); - + // int64_t lastKey = 0; // int8_t lastVal = TSDB_DATA_TINYINT_NULL; - + for (int32_t i = 0; i < numOfRow; ++i) { if (isNull((char *)&data[i], type)) { (*numOfNull) += 1; continue; } - + *sum += data[i]; if (*min > data[i]) { *min = data[i]; *minIndex = i; } - + if (*max < data[i]) { *max = data[i]; *maxIndex = i; } - + // if (type != TSDB_DATA_TYPE_BOOL) { // ignore the bool data type pre-calculation // if (isNull((char *)&lastVal, type)) { // lastKey = primaryKey[i]; @@ -3750,29 +3737,29 @@ static void getStatics_i16(int64_t *primaryKey, int16_t *data, int32_t numOfRow, *max = INT64_MIN; *minIndex = 0; *maxIndex = 0; - + assert(numOfRow <= INT16_MAX); - + // int64_t lastKey = 0; // int16_t lastVal = TSDB_DATA_SMALLINT_NULL; - + for (int32_t i = 0; i < numOfRow; ++i) { if (isNull((const char*) &data[i], TSDB_DATA_TYPE_SMALLINT)) { (*numOfNull) += 1; continue; } - + *sum += data[i]; if (*min > data[i]) { *min = data[i]; *minIndex = i; } - + if (*max < data[i]) { *max = data[i]; *maxIndex = i; } - + // if (isNull(&lastVal, TSDB_DATA_TYPE_SMALLINT)) { // lastKey = primaryKey[i]; // lastVal = data[i]; @@ -3790,29 +3777,29 @@ static void getStatics_i32(int64_t *primaryKey, int32_t *data, int32_t numOfRow, *max = INT64_MIN; *minIndex = 0; *maxIndex = 0; - + assert(numOfRow <= INT16_MAX); - + // int64_t lastKey = 0; // int32_t lastVal = TSDB_DATA_INT_NULL; - + for (int32_t i = 0; i < numOfRow; ++i) { if (isNull((const char*) &data[i], TSDB_DATA_TYPE_INT)) { (*numOfNull) += 1; continue; } - + *sum += data[i]; if (*min > data[i]) { *min = data[i]; *minIndex = i; } - + if (*max < data[i]) { *max = data[i]; *maxIndex = i; } - + // if (isNull(&lastVal, TSDB_DATA_TYPE_INT)) { // lastKey = primaryKey[i]; // lastVal = data[i]; @@ -3830,26 +3817,26 @@ static void getStatics_i64(int64_t *primaryKey, int64_t *data, int32_t numOfRow, *max = INT64_MIN; *minIndex = 0; *maxIndex = 0; - + assert(numOfRow <= INT16_MAX); - + for (int32_t i = 0; i < numOfRow; ++i) { if (isNull((const char*) &data[i], TSDB_DATA_TYPE_BIGINT)) { (*numOfNull) += 1; continue; } - + *sum += data[i]; if (*min > data[i]) { *min = data[i]; *minIndex = i; } - + if (*max < data[i]) { *max = data[i]; *maxIndex = i; } - + // if (isNull(&lastVal, TSDB_DATA_TYPE_BIGINT)) { // lastKey = primaryKey[i]; // lastVal = data[i]; @@ -3868,15 +3855,15 @@ static void getStatics_f(int64_t *primaryKey, float *data, int32_t numOfRow, dou double dsum = 0; *minIndex = 0; *maxIndex = 0; - + assert(numOfRow <= INT16_MAX); - + for (int32_t i = 0; i < numOfRow; ++i) { if (isNull((const char*) &data[i], TSDB_DATA_TYPE_FLOAT)) { (*numOfNull) += 1; continue; } - + float fv = 0; fv = GET_FLOAT_VAL(&(data[i])); dsum += fv; @@ -3884,12 +3871,12 @@ static void getStatics_f(int64_t *primaryKey, float *data, int32_t numOfRow, dou fmin = fv; *minIndex = i; } - + if (fmax < fv) { fmax = fv; *maxIndex = i; } - + // if (isNull(&lastVal, TSDB_DATA_TYPE_FLOAT)) { // lastKey = primaryKey[i]; // lastVal = data[i]; @@ -3899,7 +3886,7 @@ static void getStatics_f(int64_t *primaryKey, float *data, int32_t numOfRow, dou // lastVal = data[i]; // } } - + double csum = 0; csum = GET_DOUBLE_VAL(sum); csum += dsum; @@ -3921,15 +3908,15 @@ static void getStatics_d(int64_t *primaryKey, double *data, int32_t numOfRow, do double dsum = 0; *minIndex = 0; *maxIndex = 0; - + assert(numOfRow <= INT16_MAX); - + for (int32_t i = 0; i < numOfRow; ++i) { if (isNull((const char*) &data[i], TSDB_DATA_TYPE_DOUBLE)) { (*numOfNull) += 1; continue; } - + double dv = 0; dv = GET_DOUBLE_VAL(&(data[i])); dsum += dv; @@ -3937,12 +3924,12 @@ static void getStatics_d(int64_t *primaryKey, double *data, int32_t numOfRow, do dmin = dv; *minIndex = i; } - + if (dmax < dv) { dmax = dv; *maxIndex = i; } - + // if (isNull(&lastVal, TSDB_DATA_TYPE_DOUBLE)) { // lastKey = primaryKey[i]; // lastVal = data[i]; @@ -3952,20 +3939,20 @@ static void getStatics_d(int64_t *primaryKey, double *data, int32_t numOfRow, do // lastVal = data[i]; // } } - + double csum = 0; csum = GET_DOUBLE_VAL(sum); csum += dsum; #ifdef _TD_ARM_32_ - SET_DOUBLE_VAL_ALIGN(sum, &csum); + SET_DOUBLE_VAL_ALIGN(sum, &csum); SET_DOUBLE_VAL_ALIGN(max, &dmax); SET_DOUBLE_VAL_ALIGN(min, &dmin); #else - *sum = csum; - *max = dmax; - *min = dmin; + *sum = csum; + *max = dmax; + *min = dmin; #endif } @@ -4005,13 +3992,13 @@ static bool twa_function_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { return false; } - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes; STwaInfo * pInfo = pResInfo->interResultBuf; - + pInfo->lastKey = INT64_MIN; pInfo->type = pCtx->inputType; - + return true; } @@ -4043,69 +4030,67 @@ static FORCE_INLINE void setTWALastVal(SQLFunctionCtx *pCtx, const char *data, i static void twa_function(SQLFunctionCtx *pCtx) { void * data = GET_INPUT_CHAR(pCtx); TSKEY *primaryKey = pCtx->ptsList; - - assert(IS_DATA_BLOCK_LOADED(pCtx->blockStatus)); - + int32_t notNullElems = 0; - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo * pInfo = pResInfo->interResultBuf; - + int32_t i = 0; - + // skip null value while (pCtx->hasNull && i < pCtx->size && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) { i++; } - + if (i >= pCtx->size) { return; } - + if (pInfo->lastKey == INT64_MIN) { pInfo->lastKey = pCtx->nStartQueryTimestamp; setTWALastVal(pCtx, data, i, pInfo); - + pInfo->hasResult = DATA_SET_FLAG; } - + notNullElems++; - + if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT || pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { pInfo->dOutput += pInfo->dLastValue * (primaryKey[i] - pInfo->lastKey); } else { pInfo->iOutput += pInfo->iLastValue * (primaryKey[i] - pInfo->lastKey); } - + pInfo->lastKey = primaryKey[i]; setTWALastVal(pCtx, data, i, pInfo); - + for (++i; i < pCtx->size; i++) { if (pCtx->hasNull && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) { continue; } - + notNullElems++; if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT || pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { pInfo->dOutput += pInfo->dLastValue * (primaryKey[i] - pInfo->lastKey); } else { pInfo->iOutput += pInfo->iLastValue * (primaryKey[i] - pInfo->lastKey); } - + pInfo->lastKey = primaryKey[i]; setTWALastVal(pCtx, data, i, pInfo); } - + SET_VAL(pCtx, notNullElems, 1); - + if (notNullElems > 0) { pResInfo->hasResult = DATA_SET_FLAG; } - + if (pResInfo->superTableQ) { memcpy(pCtx->aOutputBuf, pInfo, sizeof(STwaInfo)); } - + // pCtx->numOfIteratedElems += notNullElems; } @@ -4150,33 +4135,33 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void twa_func_merge(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); assert(pResInfo->superTableQ); - + STwaInfo *pBuf = (STwaInfo *)pCtx->aOutputBuf; char * indicator = pCtx->aInputElemBuf; - + int32_t numOfNotNull = 0; for (int32_t i = 0; i < pCtx->size; ++i, indicator += sizeof(STwaInfo)) { STwaInfo *pInput = (STwaInfo*) indicator; - + if (pInput->hasResult != DATA_SET_FLAG) { continue; } - + numOfNotNull++; if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { pBuf->iOutput += pInput->iOutput; } else { pBuf->dOutput += pInput->dOutput; } - + pBuf->SKey = pInput->SKey; pBuf->EKey = pInput->EKey; pBuf->lastKey = pInput->lastKey; pBuf->iLastValue = pInput->iLastValue; } - + SET_VAL(pCtx, numOfNotNull, 1); - + if (numOfNotNull > 0) { pBuf->hasResult = DATA_SET_FLAG; } @@ -4190,22 +4175,22 @@ static void twa_func_merge(SQLFunctionCtx *pCtx) { void twa_function_copy(SQLFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + memcpy(pResInfo->interResultBuf, pCtx->aInputElemBuf, (size_t)pCtx->inputBytes); pResInfo->hasResult = ((STwaInfo *)pCtx->aInputElemBuf)->hasResult; } void twa_function_finalizer(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + STwaInfo *pInfo = (STwaInfo *)pResInfo->interResultBuf; assert(pInfo->EKey >= pInfo->lastKey && pInfo->hasResult == pResInfo->hasResult); - + if (pInfo->hasResult != DATA_SET_FLAG) { setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); return; } - + if (pInfo->SKey == pInfo->EKey) { *(double *)pCtx->aOutputBuf = 0; } else if (pInfo->type >= TSDB_DATA_TYPE_TINYINT && pInfo->type <= TSDB_DATA_TYPE_BIGINT) { @@ -4215,7 +4200,7 @@ void twa_function_finalizer(SQLFunctionCtx *pCtx) { pInfo->dOutput += pInfo->dLastValue * (pInfo->EKey - pInfo->lastKey); *(double *)pCtx->aOutputBuf = pInfo->dOutput / (pInfo->EKey - pInfo->SKey); } - + GET_RES_INFO(pCtx)->numOfRes = 1; doFinalizer(pCtx); } @@ -4238,10 +4223,10 @@ static void interp_function(SQLFunctionCtx *pCtx) { * Note: the result of primary timestamp column uses the timestamp specified by user in the query sql */ assert(pCtx->param[3].i64Key == 2); - + SInterpInfo interpInfo = *(SInterpInfo *)pCtx->aOutputBuf; SInterpInfoDetail *pInfoDetail = interpInfo.pInterpDetail; - + /* set no output result */ if (pInfoDetail->type == TSDB_INTERPO_NONE) { pCtx->param[3].i64Key = 0; @@ -4255,33 +4240,33 @@ static void interp_function(SQLFunctionCtx *pCtx) { } else if (pInfoDetail->type == TSDB_INTERPO_PREV) { char *data = pCtx->param[1].pz; char *pVal = data + TSDB_KEYSIZE; - + if (pCtx->outputType == TSDB_DATA_TYPE_FLOAT) { float v = GET_DOUBLE_VAL(pVal); assignVal(pCtx->aOutputBuf, (const char*) &v, pCtx->outputBytes, pCtx->outputType); } else { assignVal(pCtx->aOutputBuf, pVal, pCtx->outputBytes, pCtx->outputType); } - + } else if (pInfoDetail->type == TSDB_INTERPO_LINEAR) { char *data1 = pCtx->param[1].pz; char *data2 = pCtx->param[2].pz; - + char *pVal1 = data1 + TSDB_KEYSIZE; char *pVal2 = data2 + TSDB_KEYSIZE; - + SPoint point1 = {.key = *(TSKEY *)data1, .val = &pCtx->param[1].i64Key}; SPoint point2 = {.key = *(TSKEY *)data2, .val = &pCtx->param[2].i64Key}; - + SPoint point = {.key = pInfoDetail->ts, .val = pCtx->aOutputBuf}; - + int32_t srcType = pCtx->inputType; if ((srcType >= TSDB_DATA_TYPE_TINYINT && srcType <= TSDB_DATA_TYPE_BIGINT) || srcType == TSDB_DATA_TYPE_TIMESTAMP || srcType == TSDB_DATA_TYPE_DOUBLE) { point1.val = pVal1; - + point2.val = pVal2; - + if (isNull(pVal1, srcType) || isNull(pVal2, srcType)) { setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes); } else { @@ -4290,30 +4275,30 @@ static void interp_function(SQLFunctionCtx *pCtx) { } else if (srcType == TSDB_DATA_TYPE_FLOAT) { float v1 = GET_DOUBLE_VAL(pVal1); float v2 = GET_DOUBLE_VAL(pVal2); - + point1.val = &v1; point2.val = &v2; - + if (isNull(pVal1, srcType) || isNull(pVal2, srcType)) { setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes); } else { taosDoLinearInterpolation(pCtx->outputType, &point1, &point2, &point); } - + } else { setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes); } } } - + free(interpInfo.pInterpDetail); } - + pCtx->size = pCtx->param[3].i64Key; - + tVariantDestroy(&pCtx->param[1]); tVariantDestroy(&pCtx->param[2]); - + // data in the check operation are all null, not output SET_VAL(pCtx, pCtx->size, 1); } @@ -4322,10 +4307,10 @@ static bool ts_comp_function_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { return false; // not initialized since it has been initialized } - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); STSCompInfo *pInfo = pResInfo->interResultBuf; - + pInfo->pTSBuf = tsBufCreate(false); pInfo->pTSBuf->tsOrder = pCtx->order; return true; @@ -4334,9 +4319,9 @@ static bool ts_comp_function_setup(SQLFunctionCtx *pCtx) { static void ts_comp_function(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); STSBuf * pTSbuf = ((STSCompInfo *)(pResInfo->interResultBuf))->pTSBuf; - + const char *input = GET_INPUT_CHAR(pCtx); - + // primary ts must be existed, so no need to check its existance if (pCtx->order == TSQL_SO_ASC) { tsBufAppend(pTSbuf, 0, pCtx->tag.i64Key, input, pCtx->size * TSDB_KEYSIZE); @@ -4346,9 +4331,9 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) { tsBufAppend(pTSbuf, 0, pCtx->tag.i64Key, d, TSDB_KEYSIZE); } } - + SET_VAL(pCtx, pCtx->size, 1); - + pResInfo->hasResult = DATA_SET_FLAG; } @@ -4357,27 +4342,27 @@ static void ts_comp_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); STSCompInfo *pInfo = pResInfo->interResultBuf; - + STSBuf *pTSbuf = pInfo->pTSBuf; - + tsBufAppend(pTSbuf, 0, pCtx->tag.i64Key, pData, TSDB_KEYSIZE); SET_VAL(pCtx, pCtx->size, 1); - + pResInfo->hasResult = DATA_SET_FLAG; } static void ts_comp_finalize(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + STSCompInfo *pInfo = pResInfo->interResultBuf; STSBuf * pTSbuf = pInfo->pTSBuf; - + tsBufFlush(pTSbuf); strcpy(pCtx->aOutputBuf, pTSbuf->path); - + tsBufDestory(pTSbuf); doFinalizer(pCtx); } @@ -4391,7 +4376,7 @@ static double do_calc_rate(const SRateInfo* pRateInfo) { } int64_t diff = 0; - + if (pRateInfo->isIRate) { diff = pRateInfo->lastValue; if (diff >= pRateInfo->firstValue) { @@ -4403,14 +4388,14 @@ static double do_calc_rate(const SRateInfo* pRateInfo) { return 0; } } - + int64_t duration = pRateInfo->lastKey - pRateInfo->firstKey; duration = (duration + 500) / 1000; - + double resultVal = ((double)diff) / duration; - - pTrace("do_calc_rate() isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " resultVal:%f", - pRateInfo->isIRate, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, resultVal); + + pTrace("do_calc_rate() isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " resultVal:%f", + pRateInfo->isIRate, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, resultVal); return resultVal; } @@ -4420,10 +4405,10 @@ static bool rate_function_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { return false; } - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes; SRateInfo * pInfo = pResInfo->interResultBuf; - + pInfo->CorrectionValue = 0; pInfo->firstKey = INT64_MIN; pInfo->lastKey = INT64_MIN; @@ -4439,23 +4424,21 @@ static bool rate_function_setup(SQLFunctionCtx *pCtx) { static void rate_function(SQLFunctionCtx *pCtx) { - - assert(IS_DATA_BLOCK_LOADED(pCtx->blockStatus)); - + int32_t notNullElems = 0; SResultInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; TSKEY *primaryKey = pCtx->ptsList; - + pTrace("%p rate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); - + for (int32_t i = 0; i < pCtx->size; ++i) { char *pData = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { pTrace("%p rate_function() index of null data:%d", pCtx, i); continue; } - + notNullElems++; int64_t v = 0; @@ -4475,37 +4458,37 @@ static void rate_function(SQLFunctionCtx *pCtx) { default: assert(0); } - + if ((INT64_MIN == pRateInfo->firstValue) || (INT64_MIN == pRateInfo->firstKey)) { pRateInfo->firstValue = v; pRateInfo->firstKey = primaryKey[i]; - + pTrace("firstValue:%" PRId64 " firstKey:%" PRId64, pRateInfo->firstValue, pRateInfo->firstKey); } - + if (INT64_MIN == pRateInfo->lastValue) { pRateInfo->lastValue = v; } else if (v < pRateInfo->lastValue) { pRateInfo->CorrectionValue += pRateInfo->lastValue; pTrace("CorrectionValue:%" PRId64, pRateInfo->CorrectionValue); } - + pRateInfo->lastValue = v; pRateInfo->lastKey = primaryKey[i]; pTrace("lastValue:%" PRId64 " lastKey:%" PRId64, pRateInfo->lastValue, pRateInfo->lastKey); } - + if (!pCtx->hasNull) { assert(pCtx->size == notNullElems); } - + SET_VAL(pCtx, notNullElems, 1); - + if (notNullElems > 0) { pRateInfo->hasResult = DATA_SET_FLAG; pResInfo->hasResult = DATA_SET_FLAG; } - + // keep the data into the final output buffer for super table query since this execution may be the last one if (pResInfo->superTableQ) { memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo)); @@ -4517,12 +4500,12 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + // NOTE: keep the intermediate result into the interResultBuf SResultInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; TSKEY *primaryKey = pCtx->ptsList; - + int64_t v = 0; switch (pCtx->inputType) { case TSDB_DATA_TYPE_TINYINT: @@ -4544,25 +4527,25 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { if ((INT64_MIN == pRateInfo->firstValue) || (INT64_MIN == pRateInfo->firstKey)) { pRateInfo->firstValue = v; pRateInfo->firstKey = primaryKey[index]; - } + } if (INT64_MIN == pRateInfo->lastValue) { pRateInfo->lastValue = v; } else if (v < pRateInfo->lastValue) { pRateInfo->CorrectionValue += pRateInfo->lastValue; } - + pRateInfo->lastValue = v; pRateInfo->lastKey = primaryKey[index]; - + pTrace("====%p rate_function_f() index:%d lastValue:%" PRId64 " lastKey:%" PRId64 " CorrectionValue:%" PRId64, pCtx, index, pRateInfo->lastValue, pRateInfo->lastKey, pRateInfo->CorrectionValue); - + SET_VAL(pCtx, 1, 1); - + // set has result flag pRateInfo->hasResult = DATA_SET_FLAG; pResInfo->hasResult = DATA_SET_FLAG; - + // keep the data into the final output buffer for super table query since this execution may be the last one if (pResInfo->superTableQ) { memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo)); @@ -4574,30 +4557,30 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void rate_func_merge(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); assert(pResInfo->superTableQ); - + pTrace("rate_func_merge() size:%d", pCtx->size); - + //SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; SRateInfo *pBuf = (SRateInfo *)pCtx->aOutputBuf; char *indicator = pCtx->aInputElemBuf; - + assert(1 == pCtx->size); - + int32_t numOfNotNull = 0; for (int32_t i = 0; i < pCtx->size; ++i, indicator += sizeof(SRateInfo)) { SRateInfo *pInput = (SRateInfo *)indicator; if (DATA_SET_FLAG != pInput->hasResult) { continue; } - + numOfNotNull++; memcpy(pBuf, pInput, sizeof(SRateInfo)); - pTrace("%p rate_func_merge() isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64, - pCtx, pInput->isIRate, pInput->firstKey, pInput->lastKey, pInput->firstValue, pInput->lastValue, pInput->CorrectionValue); + pTrace("%p rate_func_merge() isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64, + pCtx, pInput->isIRate, pInput->firstKey, pInput->lastKey, pInput->firstValue, pInput->lastValue, pInput->CorrectionValue); } - + SET_VAL(pCtx, numOfNotNull, 1); - + if (numOfNotNull > 0) { pBuf->hasResult = DATA_SET_FLAG; } @@ -4609,14 +4592,14 @@ static void rate_func_merge(SQLFunctionCtx *pCtx) { static void rate_func_copy(SQLFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); memcpy(pResInfo->interResultBuf, pCtx->aInputElemBuf, (size_t)pCtx->inputBytes); pResInfo->hasResult = ((SRateInfo*)pCtx->aInputElemBuf)->hasResult; - + SRateInfo* pRateInfo = (SRateInfo*)pCtx->aInputElemBuf; - pTrace("%p rate_func_second_merge() firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " hasResult:%d", - pCtx, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, pRateInfo->hasResult); + pTrace("%p rate_func_second_merge() firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " hasResult:%d", + pCtx, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, pRateInfo->hasResult); } @@ -4625,9 +4608,9 @@ static void rate_finalizer(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; - pTrace("%p isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " hasResult:%d", - pCtx, pRateInfo->isIRate, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, pRateInfo->hasResult); - + pTrace("%p isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " hasResult:%d", + pCtx, pRateInfo->isIRate, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, pRateInfo->hasResult); + if (pRateInfo->hasResult != DATA_SET_FLAG) { setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); return; @@ -4636,37 +4619,35 @@ static void rate_finalizer(SQLFunctionCtx *pCtx) { *(double*)pCtx->aOutputBuf = do_calc_rate(pRateInfo); pTrace("rate_finalizer() output result:%f", *(double *)pCtx->aOutputBuf); - + // cannot set the numOfIteratedElems again since it is set during previous iteration pResInfo->numOfRes = 1; pResInfo->hasResult = DATA_SET_FLAG; - + doFinalizer(pCtx); } static void irate_function(SQLFunctionCtx *pCtx) { - - assert(IS_DATA_BLOCK_LOADED(pCtx->blockStatus)); - + int32_t notNullElems = 0; SResultInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; TSKEY *primaryKey = pCtx->ptsList; - + pTrace("%p irate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); - + if (pCtx->size < 1) { return; } - + for (int32_t i = pCtx->size - 1; i >= 0; --i) { char *pData = GET_INPUT_CHAR_INDEX(pCtx, i); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { pTrace("%p irate_function() index of null data:%d", pCtx, i); continue; } - + notNullElems++; int64_t v = 0; @@ -4686,32 +4667,32 @@ static void irate_function(SQLFunctionCtx *pCtx) { default: assert(0); } - + // TODO: calc once if only call this function once ???? if ((INT64_MIN == pRateInfo->lastKey) || (INT64_MIN == pRateInfo->lastValue)) { pRateInfo->lastValue = v; pRateInfo->lastKey = primaryKey[i]; - + pTrace("%p irate_function() lastValue:%" PRId64 " lastKey:%" PRId64, pCtx, pRateInfo->lastValue, pRateInfo->lastKey); continue; } - + if ((INT64_MIN == pRateInfo->firstKey) || (INT64_MIN == pRateInfo->firstValue)){ pRateInfo->firstValue = v; pRateInfo->firstKey = primaryKey[i]; - + pTrace("%p irate_function() firstValue:%" PRId64 " firstKey:%" PRId64, pCtx, pRateInfo->firstValue, pRateInfo->firstKey); break; } } - + SET_VAL(pCtx, notNullElems, 1); - + if (notNullElems > 0) { pRateInfo->hasResult = DATA_SET_FLAG; pResInfo->hasResult = DATA_SET_FLAG; } - + // keep the data into the final output buffer for super table query since this execution may be the last one if (pResInfo->superTableQ) { memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo)); @@ -4723,12 +4704,12 @@ static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - + // NOTE: keep the intermediate result into the interResultBuf SResultInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; TSKEY *primaryKey = pCtx->ptsList; - + int64_t v = 0; switch (pCtx->inputType) { case TSDB_DATA_TYPE_TINYINT: @@ -4746,21 +4727,21 @@ static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) { default: assert(0); } - + pRateInfo->firstKey = pRateInfo->lastKey; pRateInfo->firstValue = pRateInfo->lastValue; pRateInfo->lastValue = v; pRateInfo->lastKey = primaryKey[index]; - + pTrace("====%p irate_function_f() index:%d lastValue:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " firstKey:%" PRId64, pCtx, index, pRateInfo->lastValue, pRateInfo->lastKey, pRateInfo->firstValue , pRateInfo->firstKey); - + SET_VAL(pCtx, 1, 1); - + // set has result flag pRateInfo->hasResult = DATA_SET_FLAG; pResInfo->hasResult = DATA_SET_FLAG; - + // keep the data into the final output buffer for super table query since this execution may be the last one if (pResInfo->superTableQ) { memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo)); @@ -4770,15 +4751,15 @@ static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void do_sumrate_merge(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); assert(pResInfo->superTableQ); - + SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; char * input = GET_INPUT_CHAR(pCtx); - + for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { SRateInfo *pInput = (SRateInfo *)input; pTrace("%p do_sumrate_merge() hasResult:%d input num:%" PRId64 " input sum:%f total num:%" PRId64 " total sum:%f", pCtx, pInput->hasResult, pInput->num, pInput->sum, pRateInfo->num, pRateInfo->sum); - + if (pInput->hasResult != DATA_SET_FLAG) { continue; } else if (pInput->num == 0) { @@ -4790,7 +4771,7 @@ static void do_sumrate_merge(SQLFunctionCtx *pCtx) { } pRateInfo->hasResult = DATA_SET_FLAG; } - + // if the data set hasResult is not set, the result is null if (DATA_SET_FLAG == pRateInfo->hasResult) { pResInfo->hasResult = DATA_SET_FLAG; @@ -4812,23 +4793,23 @@ static void sumrate_func_second_merge(SQLFunctionCtx *pCtx) { static void sumrate_finalizer(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; - + pTrace("%p sumrate_finalizer() superTableQ:%d num:%" PRId64 " sum:%f hasResult:%d", pCtx, pResInfo->superTableQ, pRateInfo->num, pRateInfo->sum, pRateInfo->hasResult); - + if (pRateInfo->hasResult != DATA_SET_FLAG) { setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); return; } - + if (pRateInfo->num == 0) { - // from meter - *(double*)pCtx->aOutputBuf = do_calc_rate(pRateInfo); + // from meter + *(double*)pCtx->aOutputBuf = do_calc_rate(pRateInfo); } else if (pCtx->functionId == TSDB_FUNC_SUM_RATE || pCtx->functionId == TSDB_FUNC_SUM_IRATE) { - *(double*)pCtx->aOutputBuf = pRateInfo->sum; + *(double*)pCtx->aOutputBuf = pRateInfo->sum; } else { - *(double*)pCtx->aOutputBuf = pRateInfo->sum / pRateInfo->num; + *(double*)pCtx->aOutputBuf = pRateInfo->sum / pRateInfo->num; } - + pResInfo->numOfRes = 1; pResInfo->hasResult = DATA_SET_FLAG; doFinalizer(pCtx); @@ -4850,527 +4831,527 @@ static void sumrate_finalizer(SQLFunctionCtx *pCtx) { * */ int32_t funcCompatDefList[] = { - // count, sum, avg, min, max, stddev, percentile, apercentile, first, last - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - // last_row, top, bottom, spread, twa, leastsqr, ts, ts_dummy, tag_dummy, ts_z - 4, -1, -1, 1, 1, 1, 1, 1, 1, -1, - // tag, colprj, tagprj, arithmetic, diff, first_dist, last_dist, interp rate irate - 1, 1, 1, 1, -1, 1, 1, 5, 1, 1, - // sum_rate, sum_irate, avg_rate, avg_irate - 1, 1, 1, 1, + // count, sum, avg, min, max, stddev, percentile, apercentile, first, last + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + // last_row, top, bottom, spread, twa, leastsqr, ts, ts_dummy, tag_dummy, ts_z + 4, -1, -1, 1, 1, 1, 1, 1, 1, -1, + // tag, colprj, tagprj, arithmetic, diff, first_dist, last_dist, interp rate irate + 1, 1, 1, 1, -1, 1, 1, 5, 1, 1, + // sum_rate, sum_irate, avg_rate, avg_irate + 1, 1, 1, 1, }; SQLAggFuncElem aAggs[] = {{ - // 0, count function does not invoke the finalize function - "count", - TSDB_FUNC_COUNT, - TSDB_FUNC_COUNT, - TSDB_BASE_FUNC_SO, - function_setup, - count_function, - count_function_f, - no_next_step, - doFinalizer, - count_func_merge, - count_func_merge, - count_load_data_info, - }, - { - // 1 - "sum", - TSDB_FUNC_SUM, - TSDB_FUNC_SUM, - TSDB_BASE_FUNC_SO, - function_setup, - sum_function, - sum_function_f, - no_next_step, - function_finalizer, - sum_func_merge, - sum_func_second_merge, - precal_req_load_info, - }, - { - // 2 - "avg", - TSDB_FUNC_AVG, - TSDB_FUNC_AVG, - TSDB_BASE_FUNC_SO, - function_setup, - avg_function, - avg_function_f, - no_next_step, - avg_finalizer, - avg_func_merge, - avg_func_second_merge, - precal_req_load_info, - }, - { - // 3 - "min", - TSDB_FUNC_MIN, - TSDB_FUNC_MIN, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, - min_func_setup, - min_function, - min_function_f, - no_next_step, - function_finalizer, - min_func_merge, - min_func_second_merge, - precal_req_load_info, - }, - { - // 4 - "max", - TSDB_FUNC_MAX, - TSDB_FUNC_MAX, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, - max_func_setup, - max_function, - max_function_f, - no_next_step, - function_finalizer, - max_func_merge, - max_func_second_merge, - precal_req_load_info, - }, - { - // 5 - "stddev", - TSDB_FUNC_STDDEV, - TSDB_FUNC_INVALID_ID, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, - function_setup, - stddev_function, - stddev_function_f, - stddev_next_step, - stddev_finalizer, - noop1, - noop1, - data_req_load_info, - }, - { - // 6 - "percentile", - TSDB_FUNC_PERCT, - TSDB_FUNC_INVALID_ID, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, - percentile_function_setup, - percentile_function, - percentile_function_f, - no_next_step, - percentile_finalizer, - noop1, - noop1, - data_req_load_info, - }, - { - // 7 - "apercentile", - TSDB_FUNC_APERCT, - TSDB_FUNC_APERCT, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_METRIC, - apercentile_function_setup, - apercentile_function, - apercentile_function_f, - no_next_step, - apercentile_finalizer, - apercentile_func_merge, - apercentile_func_second_merge, - data_req_load_info, - }, - { - // 8 - "first", - TSDB_FUNC_FIRST, - TSDB_FUNC_FIRST_DST, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, - function_setup, - first_function, - first_function_f, - no_next_step, - function_finalizer, - noop1, - noop1, - first_data_req_info, - }, - { - // 9 - "last", - TSDB_FUNC_LAST, - TSDB_FUNC_LAST_DST, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, - function_setup, - last_function, - last_function_f, - no_next_step, - function_finalizer, - noop1, - noop1, - last_data_req_info, - }, - { - // 10 - "last_row", - TSDB_FUNC_LAST_ROW, - TSDB_FUNC_LAST_ROW, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS | - TSDB_FUNCSTATE_SELECTIVITY, - first_last_function_setup, - last_row_function, - noop2, - no_next_step, - last_row_finalizer, - noop1, - last_dist_func_second_merge, - data_req_load_info, - }, - { - // 11 - "top", - TSDB_FUNC_TOP, - TSDB_FUNC_TOP, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_NEED_TS | - TSDB_FUNCSTATE_SELECTIVITY, - top_bottom_function_setup, - top_function, - top_function_f, - no_next_step, - top_bottom_func_finalizer, - top_func_merge, - top_func_second_merge, - data_req_load_info, - }, - { - // 12 - "bottom", - TSDB_FUNC_BOTTOM, - TSDB_FUNC_BOTTOM, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_NEED_TS | - TSDB_FUNCSTATE_SELECTIVITY, - top_bottom_function_setup, - bottom_function, - bottom_function_f, - no_next_step, - top_bottom_func_finalizer, - bottom_func_merge, - bottom_func_second_merge, - data_req_load_info, - }, - { - // 13 - "spread", - TSDB_FUNC_SPREAD, - TSDB_FUNC_SPREAD, - TSDB_BASE_FUNC_SO, - spread_function_setup, - spread_function, - spread_function_f, - no_next_step, - spread_function_finalizer, - spread_func_merge, - spread_func_sec_merge, - count_load_data_info, - }, - { - // 14 - "twa", - TSDB_FUNC_TWA, - TSDB_FUNC_TWA, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - twa_function_setup, - twa_function, - twa_function_f, - no_next_step, - twa_function_finalizer, - twa_func_merge, - twa_function_copy, - data_req_load_info, - }, - { - // 15 - "leastsquares", - TSDB_FUNC_LEASTSQR, - TSDB_FUNC_INVALID_ID, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, - leastsquares_function_setup, - leastsquares_function, - leastsquares_function_f, - no_next_step, - leastsquares_finalizer, - noop1, - noop1, - data_req_load_info, - }, - { - // 16 - "ts", - TSDB_FUNC_TS, - TSDB_FUNC_TS, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - function_setup, - date_col_output_function, - date_col_output_function_f, - no_next_step, - doFinalizer, - copy_function, - copy_function, - no_data_info, - }, - { - // 17 - "ts", - TSDB_FUNC_TS_DUMMY, - TSDB_FUNC_TS_DUMMY, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - function_setup, - noop1, - noop2, - no_next_step, - doFinalizer, - copy_function, - copy_function, - data_req_load_info, - }, - { - // 18 - "tag", - TSDB_FUNC_TAG_DUMMY, - TSDB_FUNC_TAG_DUMMY, - TSDB_BASE_FUNC_SO, - function_setup, - tag_function, - noop2, - no_next_step, - doFinalizer, - copy_function, - copy_function, - no_data_info, - }, - { - // 19 - "ts", - TSDB_FUNC_TS_COMP, - TSDB_FUNC_TS_COMP, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_NEED_TS, - ts_comp_function_setup, - ts_comp_function, - ts_comp_function_f, - no_next_step, - ts_comp_finalize, - copy_function, - copy_function, - data_req_load_info, - }, - { - // 20 - "tag", - TSDB_FUNC_TAG, - TSDB_FUNC_TAG, - TSDB_BASE_FUNC_SO, - function_setup, - tag_function, - tag_function_f, - no_next_step, - doFinalizer, - copy_function, - copy_function, - no_data_info, - }, - { - // 21, column project sql function - "colprj", - TSDB_FUNC_PRJ, - TSDB_FUNC_PRJ, - TSDB_BASE_FUNC_MO | TSDB_FUNCSTATE_NEED_TS, - function_setup, - col_project_function, - col_project_function_f, - no_next_step, - doFinalizer, - copy_function, - copy_function, - data_req_load_info, - }, - { - // 22, multi-output, tag function has only one result - "tagprj", - TSDB_FUNC_TAGPRJ, - TSDB_FUNC_TAGPRJ, - TSDB_BASE_FUNC_MO, - function_setup, - tag_project_function, - tag_project_function_f, - no_next_step, - doFinalizer, - copy_function, - copy_function, - no_data_info, - }, - { - // 23 - "arithmetic", - TSDB_FUNC_ARITHM, - TSDB_FUNC_ARITHM, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS, - function_setup, - arithmetic_function, - arithmetic_function_f, - no_next_step, - doFinalizer, - copy_function, - copy_function, - data_req_load_info, - }, - { - // 24 - "diff", - TSDB_FUNC_DIFF, - TSDB_FUNC_INVALID_ID, - TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_NEED_TS, - diff_function_setup, - diff_function, - diff_function_f, - no_next_step, - doFinalizer, - noop1, - noop1, - data_req_load_info, - }, - // distributed version used in two-stage aggregation processes - { - // 25 - "first_dist", - TSDB_FUNC_FIRST_DST, - TSDB_FUNC_FIRST_DST, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, - first_last_function_setup, - first_dist_function, - first_dist_function_f, - no_next_step, - function_finalizer, - first_dist_func_merge, - first_dist_func_second_merge, - first_dist_data_req_info, - }, - { - // 26 - "last_dist", - TSDB_FUNC_LAST_DST, - TSDB_FUNC_LAST_DST, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, - first_last_function_setup, - last_dist_function, - last_dist_function_f, - no_next_step, - function_finalizer, - last_dist_func_merge, - last_dist_func_second_merge, - last_dist_data_req_info, - }, - { - // 27 - "interp", - TSDB_FUNC_INTERP, - TSDB_FUNC_INTERP, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS, - function_setup, - interp_function, - do_sum_f, // todo filter handle - no_next_step, - doFinalizer, - noop1, - copy_function, - no_data_info, - }, - { - // 28 - "rate", - TSDB_FUNC_RATE, - TSDB_FUNC_RATE, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - rate_function_setup, - rate_function, - rate_function_f, - no_next_step, - rate_finalizer, - rate_func_merge, - rate_func_copy, - data_req_load_info, - }, - { - // 29 - "irate", - TSDB_FUNC_IRATE, - TSDB_FUNC_IRATE, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - rate_function_setup, - irate_function, - irate_function_f, - no_next_step, - rate_finalizer, - rate_func_merge, - rate_func_copy, - data_req_load_info, - }, - { - // 30 - "sum_rate", - TSDB_FUNC_SUM_RATE, - TSDB_FUNC_SUM_RATE, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - rate_function_setup, - rate_function, - rate_function_f, - no_next_step, - sumrate_finalizer, - sumrate_func_merge, - sumrate_func_second_merge, - data_req_load_info, - }, - { - // 31 - "sum_irate", - TSDB_FUNC_SUM_IRATE, - TSDB_FUNC_SUM_IRATE, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - rate_function_setup, - irate_function, - irate_function_f, - no_next_step, - sumrate_finalizer, - sumrate_func_merge, - sumrate_func_second_merge, - data_req_load_info, - }, - { - // 32 - "avg_rate", - TSDB_FUNC_AVG_RATE, - TSDB_FUNC_AVG_RATE, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - rate_function_setup, - rate_function, - rate_function_f, - no_next_step, - sumrate_finalizer, - sumrate_func_merge, - sumrate_func_second_merge, - data_req_load_info, - }, - { - // 33 - "avg_irate", - TSDB_FUNC_AVG_IRATE, - TSDB_FUNC_AVG_IRATE, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - rate_function_setup, - irate_function, - irate_function_f, - no_next_step, - sumrate_finalizer, - sumrate_func_merge, - sumrate_func_second_merge, - data_req_load_info, - }}; + // 0, count function does not invoke the finalize function + "count", + TSDB_FUNC_COUNT, + TSDB_FUNC_COUNT, + TSDB_BASE_FUNC_SO, + function_setup, + count_function, + count_function_f, + no_next_step, + doFinalizer, + count_func_merge, + count_func_merge, + count_load_data_info, + }, + { + // 1 + "sum", + TSDB_FUNC_SUM, + TSDB_FUNC_SUM, + TSDB_BASE_FUNC_SO, + function_setup, + sum_function, + sum_function_f, + no_next_step, + function_finalizer, + sum_func_merge, + sum_func_second_merge, + precal_req_load_info, + }, + { + // 2 + "avg", + TSDB_FUNC_AVG, + TSDB_FUNC_AVG, + TSDB_BASE_FUNC_SO, + function_setup, + avg_function, + avg_function_f, + no_next_step, + avg_finalizer, + avg_func_merge, + avg_func_second_merge, + precal_req_load_info, + }, + { + // 3 + "min", + TSDB_FUNC_MIN, + TSDB_FUNC_MIN, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, + min_func_setup, + min_function, + min_function_f, + no_next_step, + function_finalizer, + min_func_merge, + min_func_second_merge, + precal_req_load_info, + }, + { + // 4 + "max", + TSDB_FUNC_MAX, + TSDB_FUNC_MAX, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, + max_func_setup, + max_function, + max_function_f, + no_next_step, + function_finalizer, + max_func_merge, + max_func_second_merge, + precal_req_load_info, + }, + { + // 5 + "stddev", + TSDB_FUNC_STDDEV, + TSDB_FUNC_INVALID_ID, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, + function_setup, + stddev_function, + stddev_function_f, + stddev_next_step, + stddev_finalizer, + noop1, + noop1, + data_req_load_info, + }, + { + // 6 + "percentile", + TSDB_FUNC_PERCT, + TSDB_FUNC_INVALID_ID, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, + percentile_function_setup, + percentile_function, + percentile_function_f, + no_next_step, + percentile_finalizer, + noop1, + noop1, + data_req_load_info, + }, + { + // 7 + "apercentile", + TSDB_FUNC_APERCT, + TSDB_FUNC_APERCT, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_METRIC, + apercentile_function_setup, + apercentile_function, + apercentile_function_f, + no_next_step, + apercentile_finalizer, + apercentile_func_merge, + apercentile_func_second_merge, + data_req_load_info, + }, + { + // 8 + "first", + TSDB_FUNC_FIRST, + TSDB_FUNC_FIRST_DST, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, + function_setup, + first_function, + first_function_f, + no_next_step, + function_finalizer, + noop1, + noop1, + first_data_req_info, + }, + { + // 9 + "last", + TSDB_FUNC_LAST, + TSDB_FUNC_LAST_DST, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, + function_setup, + last_function, + last_function_f, + no_next_step, + function_finalizer, + noop1, + noop1, + last_data_req_info, + }, + { + // 10 + "last_row", + TSDB_FUNC_LAST_ROW, + TSDB_FUNC_LAST_ROW, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS | + TSDB_FUNCSTATE_SELECTIVITY, + first_last_function_setup, + last_row_function, + noop2, + no_next_step, + last_row_finalizer, + noop1, + last_dist_func_second_merge, + data_req_load_info, + }, + { + // 11 + "top", + TSDB_FUNC_TOP, + TSDB_FUNC_TOP, + TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_NEED_TS | + TSDB_FUNCSTATE_SELECTIVITY, + top_bottom_function_setup, + top_function, + top_function_f, + no_next_step, + top_bottom_func_finalizer, + top_func_merge, + top_func_second_merge, + data_req_load_info, + }, + { + // 12 + "bottom", + TSDB_FUNC_BOTTOM, + TSDB_FUNC_BOTTOM, + TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_NEED_TS | + TSDB_FUNCSTATE_SELECTIVITY, + top_bottom_function_setup, + bottom_function, + bottom_function_f, + no_next_step, + top_bottom_func_finalizer, + bottom_func_merge, + bottom_func_second_merge, + data_req_load_info, + }, + { + // 13 + "spread", + TSDB_FUNC_SPREAD, + TSDB_FUNC_SPREAD, + TSDB_BASE_FUNC_SO, + spread_function_setup, + spread_function, + spread_function_f, + no_next_step, + spread_function_finalizer, + spread_func_merge, + spread_func_sec_merge, + count_load_data_info, + }, + { + // 14 + "twa", + TSDB_FUNC_TWA, + TSDB_FUNC_TWA, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, + twa_function_setup, + twa_function, + twa_function_f, + no_next_step, + twa_function_finalizer, + twa_func_merge, + twa_function_copy, + data_req_load_info, + }, + { + // 15 + "leastsquares", + TSDB_FUNC_LEASTSQR, + TSDB_FUNC_INVALID_ID, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, + leastsquares_function_setup, + leastsquares_function, + leastsquares_function_f, + no_next_step, + leastsquares_finalizer, + noop1, + noop1, + data_req_load_info, + }, + { + // 16 + "ts", + TSDB_FUNC_TS, + TSDB_FUNC_TS, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, + function_setup, + date_col_output_function, + date_col_output_function_f, + no_next_step, + doFinalizer, + copy_function, + copy_function, + no_data_info, + }, + { + // 17 + "ts", + TSDB_FUNC_TS_DUMMY, + TSDB_FUNC_TS_DUMMY, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, + function_setup, + noop1, + noop2, + no_next_step, + doFinalizer, + copy_function, + copy_function, + data_req_load_info, + }, + { + // 18 + "tag", + TSDB_FUNC_TAG_DUMMY, + TSDB_FUNC_TAG_DUMMY, + TSDB_BASE_FUNC_SO, + function_setup, + tag_function, + noop2, + no_next_step, + doFinalizer, + copy_function, + copy_function, + no_data_info, + }, + { + // 19 + "ts", + TSDB_FUNC_TS_COMP, + TSDB_FUNC_TS_COMP, + TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_NEED_TS, + ts_comp_function_setup, + ts_comp_function, + ts_comp_function_f, + no_next_step, + ts_comp_finalize, + copy_function, + copy_function, + data_req_load_info, + }, + { + // 20 + "tag", + TSDB_FUNC_TAG, + TSDB_FUNC_TAG, + TSDB_BASE_FUNC_SO, + function_setup, + tag_function, + tag_function_f, + no_next_step, + doFinalizer, + copy_function, + copy_function, + no_data_info, + }, + { + // 21, column project sql function + "colprj", + TSDB_FUNC_PRJ, + TSDB_FUNC_PRJ, + TSDB_BASE_FUNC_MO | TSDB_FUNCSTATE_NEED_TS, + function_setup, + col_project_function, + col_project_function_f, + no_next_step, + doFinalizer, + copy_function, + copy_function, + data_req_load_info, + }, + { + // 22, multi-output, tag function has only one result + "tagprj", + TSDB_FUNC_TAGPRJ, + TSDB_FUNC_TAGPRJ, + TSDB_BASE_FUNC_MO, + function_setup, + tag_project_function, + tag_project_function_f, + no_next_step, + doFinalizer, + copy_function, + copy_function, + no_data_info, + }, + { + // 23 + "arithmetic", + TSDB_FUNC_ARITHM, + TSDB_FUNC_ARITHM, + TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS, + function_setup, + arithmetic_function, + arithmetic_function_f, + no_next_step, + doFinalizer, + copy_function, + copy_function, + data_req_load_info, + }, + { + // 24 + "diff", + TSDB_FUNC_DIFF, + TSDB_FUNC_INVALID_ID, + TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_NEED_TS, + diff_function_setup, + diff_function, + diff_function_f, + no_next_step, + doFinalizer, + noop1, + noop1, + data_req_load_info, + }, + // distributed version used in two-stage aggregation processes + { + // 25 + "first_dist", + TSDB_FUNC_FIRST_DST, + TSDB_FUNC_FIRST_DST, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, + first_last_function_setup, + first_dist_function, + first_dist_function_f, + no_next_step, + function_finalizer, + first_dist_func_merge, + first_dist_func_second_merge, + first_dist_data_req_info, + }, + { + // 26 + "last_dist", + TSDB_FUNC_LAST_DST, + TSDB_FUNC_LAST_DST, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, + first_last_function_setup, + last_dist_function, + last_dist_function_f, + no_next_step, + function_finalizer, + last_dist_func_merge, + last_dist_func_second_merge, + last_dist_data_req_info, + }, + { + // 27 + "interp", + TSDB_FUNC_INTERP, + TSDB_FUNC_INTERP, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_NEED_TS, + function_setup, + interp_function, + do_sum_f, // todo filter handle + no_next_step, + doFinalizer, + noop1, + copy_function, + no_data_info, + }, + { + // 28 + "rate", + TSDB_FUNC_RATE, + TSDB_FUNC_RATE, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, + rate_function_setup, + rate_function, + rate_function_f, + no_next_step, + rate_finalizer, + rate_func_merge, + rate_func_copy, + data_req_load_info, + }, + { + // 29 + "irate", + TSDB_FUNC_IRATE, + TSDB_FUNC_IRATE, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, + rate_function_setup, + irate_function, + irate_function_f, + no_next_step, + rate_finalizer, + rate_func_merge, + rate_func_copy, + data_req_load_info, + }, + { + // 30 + "sum_rate", + TSDB_FUNC_SUM_RATE, + TSDB_FUNC_SUM_RATE, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, + rate_function_setup, + rate_function, + rate_function_f, + no_next_step, + sumrate_finalizer, + sumrate_func_merge, + sumrate_func_second_merge, + data_req_load_info, + }, + { + // 31 + "sum_irate", + TSDB_FUNC_SUM_IRATE, + TSDB_FUNC_SUM_IRATE, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, + rate_function_setup, + irate_function, + irate_function_f, + no_next_step, + sumrate_finalizer, + sumrate_func_merge, + sumrate_func_second_merge, + data_req_load_info, + }, + { + // 32 + "avg_rate", + TSDB_FUNC_AVG_RATE, + TSDB_FUNC_AVG_RATE, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, + rate_function_setup, + rate_function, + rate_function_f, + no_next_step, + sumrate_finalizer, + sumrate_func_merge, + sumrate_func_second_merge, + data_req_load_info, + }, + { + // 33 + "avg_irate", + TSDB_FUNC_AVG_IRATE, + TSDB_FUNC_AVG_IRATE, + TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, + rate_function_setup, + irate_function, + irate_function_f, + no_next_step, + sumrate_finalizer, + sumrate_func_merge, + sumrate_func_second_merge, + data_req_load_info, + }}; diff --git a/src/client/src/tscJoinProcess.c b/src/client/src/tscJoinProcess.c deleted file mode 100644 index 882c72ae8019c59ff2764f43a201c4c23086f2b1..0000000000000000000000000000000000000000 --- a/src/client/src/tscJoinProcess.c +++ /dev/null @@ -1,812 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "tscJoinProcess.h" -#include "tsclient.h" -#include "qtsbuf.h" - -static void freeSubqueryObj(SSqlObj* pSql); - -static bool doCompare(int32_t order, int64_t left, int64_t right) { - if (order == TSQL_SO_ASC) { - return left < right; - } else { - return left > right; - } -} - -static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSubquerySupporter* pSupporter1, - SJoinSubquerySupporter* pSupporter2, TSKEY* st, TSKEY* et) { - STSBuf* output1 = tsBufCreate(true); - STSBuf* output2 = tsBufCreate(true); - - *st = INT64_MAX; - *et = INT64_MIN; - - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); - - SLimitVal* pLimit = &pQueryInfo->limit; - int32_t order = pQueryInfo->order.order; - - SQueryInfo* pSubQueryInfo1 = tscGetQueryInfoDetail(&pSql->pSubs[0]->cmd, 0); - SQueryInfo* pSubQueryInfo2 = tscGetQueryInfoDetail(&pSql->pSubs[1]->cmd, 0); - - pSubQueryInfo1->tsBuf = output1; - pSubQueryInfo2->tsBuf = output2; - - tsBufResetPos(pSupporter1->pTSBuf); - tsBufResetPos(pSupporter2->pTSBuf); - - // TODO add more details information - if (!tsBufNextPos(pSupporter1->pTSBuf)) { - tsBufFlush(output1); - tsBufFlush(output2); - - tscTrace("%p input1 is empty, 0 for secondary query after ts blocks intersecting", pSql); - return 0; - } - - if (!tsBufNextPos(pSupporter2->pTSBuf)) { - tsBufFlush(output1); - tsBufFlush(output2); - - tscTrace("%p input2 is empty, 0 for secondary query after ts blocks intersecting", pSql); - return 0; - } - - int64_t numOfInput1 = 1; - int64_t numOfInput2 = 1; - - while (1) { - STSElem elem1 = tsBufGetElem(pSupporter1->pTSBuf); - STSElem elem2 = tsBufGetElem(pSupporter2->pTSBuf); - -#ifdef _DEBUG_VIEW - // for debug purpose - tscPrint("%" PRId64 ", tags:%d \t %" PRId64 ", tags:%d", elem1.ts, elem1.tag, elem2.ts, elem2.tag); -#endif - - if (elem1.tag < elem2.tag || (elem1.tag == elem2.tag && doCompare(order, elem1.ts, elem2.ts))) { - if (!tsBufNextPos(pSupporter1->pTSBuf)) { - break; - } - - numOfInput1++; - } else if (elem1.tag > elem2.tag || (elem1.tag == elem2.tag && doCompare(order, elem2.ts, elem1.ts))) { - if (!tsBufNextPos(pSupporter2->pTSBuf)) { - break; - } - - numOfInput2++; - } else { - /* - * in case of stable query, limit/offset is not applied here. the limit/offset is applied to the - * final results which is acquired after the secondry merge of in the client. - */ - if (pLimit->offset == 0 || pQueryInfo->intervalTime > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) { - if (*st > elem1.ts) { - *st = elem1.ts; - } - - if (*et < elem1.ts) { - *et = elem1.ts; - } - - tsBufAppend(output1, elem1.vnode, elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts)); - tsBufAppend(output2, elem2.vnode, elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts)); - } else { - pLimit->offset -= 1; - } - - if (!tsBufNextPos(pSupporter1->pTSBuf)) { - break; - } - - numOfInput1++; - - if (!tsBufNextPos(pSupporter2->pTSBuf)) { - break; - } - - numOfInput2++; - } - } - - /* - * failed to set the correct ts order yet in two cases: - * 1. only one element - * 2. only one element for each tag. - */ - if (output1->tsOrder == -1) { - output1->tsOrder = TSQL_SO_ASC; - output2->tsOrder = TSQL_SO_ASC; - } - - tsBufFlush(output1); - tsBufFlush(output2); - - tsBufDestory(pSupporter1->pTSBuf); - tsBufDestory(pSupporter2->pTSBuf); - - tscTrace("%p input1:%" PRId64 ", input2:%" PRId64 ", final:%" PRId64 " for secondary query after ts blocks " - "intersecting, skey:%" PRId64 ", ekey:%" PRId64, pSql, - numOfInput1, numOfInput2, output1->numOfTotal, *st, *et); - - return output1->numOfTotal; -} - -// todo handle failed to create sub query -SJoinSubquerySupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, int32_t index) { - SJoinSubquerySupporter* pSupporter = calloc(1, sizeof(SJoinSubquerySupporter)); - if (pSupporter == NULL) { - return NULL; - } - - pSupporter->pObj = pSql; - pSupporter->pState = pState; - - pSupporter->subqueryIndex = index; - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); - - pSupporter->interval = pQueryInfo->intervalTime; - pSupporter->limit = pQueryInfo->limit; - - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, pSql->cmd.clauseIndex, index); - pSupporter->uid = pMeterMetaInfo->pMeterMeta->uid; - - assert (pSupporter->uid != 0); - - getTmpfilePath("join-", pSupporter->path); - pSupporter->f = fopen(pSupporter->path, "w"); - - if (pSupporter->f == NULL) { - tscError("%p failed to create tmp file:%s, reason:%s", pSql, pSupporter->path, strerror(errno)); - } - - return pSupporter; -} - -void tscDestroyJoinSupporter(SJoinSubquerySupporter* pSupporter) { - if (pSupporter == NULL) { - return; - } - - tscSqlExprInfoDestroy(&pSupporter->exprsInfo); - tscColumnBaseInfoDestroy(&pSupporter->colList); - - tscClearFieldInfo(&pSupporter->fieldsInfo); - - if (pSupporter->f != NULL) { - fclose(pSupporter->f); - unlink(pSupporter->path); - } - - tscTagCondRelease(&pSupporter->tagCond); - free(pSupporter); -} - -/* - * need the secondary query process - * In case of count(ts)/count(*)/spread(ts) query, that are only applied to - * primary timestamp column , the secondary query is not necessary - * - */ -bool needSecondaryQuery(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->colList.numOfCols; ++i) { - SColumnBase* pBase = tscColumnBaseInfoGet(&pQueryInfo->colList, i); - if (pBase->colIndex.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return true; - } - } - - return false; -} - -/* - * launch secondary stage query to fetch the result that contains timestamp in set - */ -int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { - int32_t numOfSub = 0; - SJoinSubquerySupporter* pSupporter = NULL; - - /* - * If the columns are not involved in the final select clause, the secondary query will not be launched - * for the subquery. - */ - SSubqueryState* pState = NULL; - - for (int32_t i = 0; i < pSql->numOfSubs; ++i) { - pSupporter = pSql->pSubs[i]->param; - if (pSupporter->exprsInfo.numOfExprs > 0) { - ++numOfSub; - } - } - - assert(numOfSub > 0); - - // scan all subquery, if one sub query has only ts, ignore it - tscTrace("%p start to launch secondary subqueries, total:%d, only:%d needs to query, others are not retrieve in " - "select clause", pSql, pSql->numOfSubs, numOfSub); - - /* - * the subqueries that do not actually launch the secondary query to virtual node is set as completed. - */ - pState = pSupporter->pState; - pState->numOfTotal = pSql->numOfSubs; - pState->numOfCompleted = (pSql->numOfSubs - numOfSub); - - bool success = true; - - for (int32_t i = 0; i < pSql->numOfSubs; ++i) { - SSqlObj *pPrevSub = pSql->pSubs[i]; - pSql->pSubs[i] = NULL; - - pSupporter = pPrevSub->param; - - if (pSupporter->exprsInfo.numOfExprs == 0) { - tscTrace("%p subIndex: %d, not need to launch query, ignore it", pSql, i); - - tscDestroyJoinSupporter(pSupporter); - tscFreeSqlObj(pPrevSub); - - pSql->pSubs[i] = NULL; - continue; - } - - SQueryInfo *pSubQueryInfo = tscGetQueryInfoDetail(&pPrevSub->cmd, 0); - STSBuf *pTSBuf = pSubQueryInfo->tsBuf; - pSubQueryInfo->tsBuf = NULL; - - // free result for async object will also free sqlObj - assert(pSubQueryInfo->exprsInfo.numOfExprs == 1); // ts_comp query only requires one resutl columns - taos_free_result(pPrevSub); - - SSqlObj *pNew = createSubqueryObj(pSql, (int16_t) i, tscJoinQueryCallback, pSupporter, NULL); - if (pNew == NULL) { - tscDestroyJoinSupporter(pSupporter); - success = false; - break; - } - - tscClearSubqueryInfo(&pNew->cmd); - pSql->pSubs[i] = pNew; - - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); - pQueryInfo->tsBuf = pTSBuf; // transfer the ownership of timestamp comp-z data to the new created object - - // set the second stage sub query for join process - pQueryInfo->type |= TSDB_QUERY_TYPE_JOIN_SEC_STAGE; - - pQueryInfo->intervalTime = pSupporter->interval; - pQueryInfo->groupbyExpr = pSupporter->groupbyExpr; - - tscColumnBaseInfoCopy(&pQueryInfo->colList, &pSupporter->colList, 0); - tscTagCondCopy(&pQueryInfo->tagCond, &pSupporter->tagCond); - - tscSqlExprCopy(&pQueryInfo->exprsInfo, &pSupporter->exprsInfo, pSupporter->uid, false); - tscFieldInfoCopyAll(&pQueryInfo->fieldsInfo, &pSupporter->fieldsInfo); - - pSupporter->exprsInfo.numOfExprs = 0; - pSupporter->fieldsInfo.numOfOutputCols = 0; - - /* - * if the first column of the secondary query is not ts function, add this function. - * Because this column is required to filter with timestamp after intersecting. - */ - if (pSupporter->exprsInfo.pExprs[0]->functionId != TSDB_FUNC_TS) { - tscAddTimestampColumn(pQueryInfo, TSDB_FUNC_TS, 0); - } - - SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); - assert(pNew->numOfSubs == 0 && pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1); - - tscFieldInfoCalOffset(pNewQueryInfo); - - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pNewQueryInfo, 0); - - /* - * When handling the projection query, the offset value will be modified for table-table join, which is changed - * during the timestamp intersection. - */ - pSupporter->limit = pQueryInfo->limit; - pNewQueryInfo->limit = pSupporter->limit; - - // fetch the join tag column - if (UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) { - SSqlExpr *pExpr = tscSqlExprGet(pNewQueryInfo, 0); - assert(pQueryInfo->tagCond.joinInfo.hasJoin); - - int16_t tagColIndex = tscGetJoinTagColIndexByUid(&pQueryInfo->tagCond, pMeterMetaInfo->pMeterMeta->uid); - pExpr->param[0].i64Key = tagColIndex; - pExpr->numOfParams = 1; - } - - tscPrintSelectClause(pNew, 0); - - tscTrace("%p subquery:%p tableIndex:%d, vnodeIdx:%d, type:%d, exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s", - pSql, pNew, 0, pMeterMetaInfo->vnodeIndex, pNewQueryInfo->type, - pNewQueryInfo->exprsInfo.numOfExprs, pNewQueryInfo->colList.numOfCols, - pNewQueryInfo->fieldsInfo.numOfOutputCols, pNewQueryInfo->pMeterInfo[0]->name); - } - - //prepare the subqueries object failed, abort - if (!success) { - pSql->res.code = TSDB_CODE_CLI_OUT_OF_MEMORY; - tscError("%p failed to prepare subqueries objs for secondary phase query, numOfSub:%d, code:%d", pSql, - pSql->numOfSubs, pSql->res.code); - freeSubqueryObj(pSql); - - return pSql->res.code; - } - - for(int32_t i = 0; i < pSql->numOfSubs; ++i) { - SSqlObj* pSub = pSql->pSubs[i]; - if (pSub == NULL) { - continue; - } - - tscProcessSql(pSub); - } - - return TSDB_CODE_SUCCESS; -} - -static void freeSubqueryObj(SSqlObj* pSql) { - SSubqueryState* pState = NULL; - - for (int32_t i = 0; i < pSql->numOfSubs; ++i) { - if (pSql->pSubs[i] != NULL) { - SJoinSubquerySupporter* p = pSql->pSubs[i]->param; - pState = p->pState; - - tscDestroyJoinSupporter(p); - - if (pSql->pSubs[i]->res.code == TSDB_CODE_SUCCESS) { - taos_free_result(pSql->pSubs[i]); - } - } - } - - tfree(pState); - pSql->numOfSubs = 0; -} - -static void doQuitSubquery(SSqlObj* pParentSql) { - freeSubqueryObj(pParentSql); - - tsem_wait(&pParentSql->emptyRspSem); - tsem_wait(&pParentSql->emptyRspSem); - - tsem_post(&pParentSql->rspSem); -} - -static void quitAllSubquery(SSqlObj* pSqlObj, SJoinSubquerySupporter* pSupporter) { - int32_t numOfTotal = pSupporter->pState->numOfTotal; - int32_t finished = atomic_add_fetch_32(&pSupporter->pState->numOfCompleted, 1); - - if (finished >= numOfTotal) { - pSqlObj->res.code = abs(pSupporter->pState->code); - tscError("%p all subquery return and query failed, global code:%d", pSqlObj, pSqlObj->res.code); - - doQuitSubquery(pSqlObj); - } -} - -// update the query time range according to the join results on timestamp -static void updateQueryTimeRange(SQueryInfo* pQueryInfo, int64_t st, int64_t et) { - assert(pQueryInfo->stime <= st && pQueryInfo->etime >= et); - - pQueryInfo->stime = st; - pQueryInfo->etime = et; -} - -static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { - SJoinSubquerySupporter* pSupporter = (SJoinSubquerySupporter*)param; - SSqlObj* pParentSql = pSupporter->pObj; - - SSqlObj* pSql = (SSqlObj*)tres; - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - - if ((pQueryInfo->type & TSDB_QUERY_TYPE_JOIN_SEC_STAGE) == 0) { - if (pSupporter->pState->code != TSDB_CODE_SUCCESS) { - tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, numOfRows, - pSupporter->pState->code); - - quitAllSubquery(pParentSql, pSupporter); - return; - } - - if (numOfRows > 0) { // write the data into disk - fwrite(pSql->res.data, pSql->res.numOfRows, 1, pSupporter->f); - fclose(pSupporter->f); - - STSBuf* pBuf = tsBufCreateFromFile(pSupporter->path, true); - if (pBuf == NULL) { - tscError("%p invalid ts comp file from vnode, abort sub query, file size:%d", pSql, numOfRows); - - pSupporter->pState->code = TSDB_CODE_APP_ERROR; // todo set the informative code - quitAllSubquery(pParentSql, pSupporter); - return; - } - - if (pSupporter->pTSBuf == NULL) { - tscTrace("%p create tmp file for ts block:%s", pSql, pBuf->path); - pSupporter->pTSBuf = pBuf; - } else { - assert(pQueryInfo->numOfTables == 1); // for subquery, only one metermetaInfo - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - - tsBufMerge(pSupporter->pTSBuf, pBuf, pMeterMetaInfo->vnodeIndex); - tsBufDestory(pBuf); - } - - // open new file to save the result - getTmpfilePath("ts-join", pSupporter->path); - pSupporter->f = fopen(pSupporter->path, "w"); - pSql->res.row = pSql->res.numOfRows; - - taos_fetch_rows_a(tres, joinRetrieveCallback, param); - } else if (numOfRows == 0) { // no data from this vnode anymore - SQueryInfo* pParentQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, pParentSql->cmd.clauseIndex); - - //todo refactor - if (tscNonOrderedProjectionQueryOnSTable(pParentQueryInfo, 0)) { - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - assert(pQueryInfo->numOfTables == 1); - - // for projection query, need to try next vnode - int32_t totalVnode = pMeterMetaInfo->pMetricMeta->numOfVnodes; - if ((++pMeterMetaInfo->vnodeIndex) < totalVnode) { - tscTrace("%p current vnode:%d exhausted, try next:%d. total vnode:%d. current numOfRes:%d", pSql, - pMeterMetaInfo->vnodeIndex - 1, pMeterMetaInfo->vnodeIndex, totalVnode, pRes->numOfTotal); - - pSql->cmd.command = TSDB_SQL_SELECT; - pSql->fp = tscJoinQueryCallback; - tscProcessSql(pSql); - - return; - } - } - - int32_t numOfTotal = pSupporter->pState->numOfTotal; - int32_t finished = atomic_add_fetch_32(&pSupporter->pState->numOfCompleted, 1); - - if (finished >= numOfTotal) { - assert(finished == numOfTotal); - - if (pSupporter->pState->code != TSDB_CODE_SUCCESS) { - tscTrace("%p sub:%p, numOfSub:%d, quit from further procedure due to other queries failure", pParentSql, tres, - pSupporter->subqueryIndex); - doQuitSubquery(pParentSql); - return; - } - - tscTrace("%p all subqueries retrieve ts complete, do ts block intersect", pParentSql); - - SJoinSubquerySupporter* p1 = pParentSql->pSubs[0]->param; - SJoinSubquerySupporter* p2 = pParentSql->pSubs[1]->param; - - TSKEY st, et; - - int64_t num = doTSBlockIntersect(pParentSql, p1, p2, &st, &et); - if (num <= 0) { // no result during ts intersect - tscTrace("%p free all sub SqlObj and quit", pParentSql); - doQuitSubquery(pParentSql); - } else { - updateQueryTimeRange(pParentQueryInfo, st, et); - tscLaunchSecondPhaseSubqueries(pParentSql); - } - } - } else { // failure of sub query - tscError("%p sub query failed, code:%d, index:%d", pSql, numOfRows, pSupporter->subqueryIndex); - pSupporter->pState->code = numOfRows; - - quitAllSubquery(pParentSql, pSupporter); - return; - } - - } else { // secondary stage retrieve, driven by taos_fetch_row or other functions - if (numOfRows < 0) { - pSupporter->pState->code = numOfRows; - tscError("%p retrieve failed, code:%d, index:%d", pSql, numOfRows, pSupporter->subqueryIndex); - } - - if (numOfRows >= 0) { - pSql->res.numOfTotal += pSql->res.numOfRows; - } - - if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && numOfRows == 0) { - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - assert(pQueryInfo->numOfTables == 1); - - // for projection query, need to try next vnode if current vnode is exhausted - if ((++pMeterMetaInfo->vnodeIndex) < pMeterMetaInfo->pMetricMeta->numOfVnodes) { - pSupporter->pState->numOfCompleted = 0; - pSupporter->pState->numOfTotal = 1; - - pSql->cmd.command = TSDB_SQL_SELECT; - pSql->fp = tscJoinQueryCallback; - tscProcessSql(pSql); - - return; - } - } - - int32_t numOfTotal = pSupporter->pState->numOfTotal; - int32_t finished = atomic_add_fetch_32(&pSupporter->pState->numOfCompleted, 1); - - if (finished >= numOfTotal) { - assert(finished == numOfTotal); - tscTrace("%p all %d secondary subquery retrieves completed, global code:%d", tres, numOfTotal, - pParentSql->res.code); - - if (pSupporter->pState->code != TSDB_CODE_SUCCESS) { - pParentSql->res.code = abs(pSupporter->pState->code); - freeSubqueryObj(pParentSql); - } - - tsem_post(&pParentSql->rspSem); - } else { - tscTrace("%p sub:%p completed, completed:%d, total:%d", pParentSql, tres, finished, numOfTotal); - } - } -} - -static SJoinSubquerySupporter* tscUpdateSubqueryStatus(SSqlObj* pSql, int32_t numOfFetch) { - int32_t notInvolved = 0; - SJoinSubquerySupporter* pSupporter = NULL; - SSubqueryState* pState = NULL; - - for(int32_t i = 0; i < pSql->numOfSubs; ++i) { - if (pSql->pSubs[i] == NULL) { - notInvolved++; - } else { - pSupporter = (SJoinSubquerySupporter*)pSql->pSubs[i]->param; - pState = pSupporter->pState; - } - } - - pState->numOfTotal = pSql->numOfSubs; - pState->numOfCompleted = pSql->numOfSubs - numOfFetch; - - return pSupporter; -} - -void tscFetchDatablockFromSubquery(SSqlObj* pSql) { - int32_t numOfFetch = 0; - assert(pSql->numOfSubs >= 1); - - for (int32_t i = 0; i < pSql->numOfSubs; ++i) { - if (pSql->pSubs[i] == NULL) { // this subquery does not need to involve in secondary query - continue; - } - - SSqlRes *pRes = &pSql->pSubs[i]->res; - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->pSubs[i]->cmd, 0); - - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - - if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { - if (pRes->row >= pRes->numOfRows && pMeterMetaInfo->vnodeIndex < pMeterMetaInfo->pMetricMeta->numOfVnodes && - (!tscHasReachLimitation(pQueryInfo, pRes))) { - numOfFetch++; - } - } else { - if (pRes->row >= pRes->numOfRows && (!tscHasReachLimitation(pQueryInfo, pRes))) { - numOfFetch++; - } - } - } - - if (numOfFetch <= 0) { - return; - } - - // TODO multi-vnode retrieve for projection query with limitation has bugs, since the global limiation is not handled - tscTrace("%p retrieve data from %d subqueries", pSql, numOfFetch); - - SJoinSubquerySupporter* pSupporter = tscUpdateSubqueryStatus(pSql, numOfFetch); - - for (int32_t i = 0; i < pSql->numOfSubs; ++i) { - SSqlObj* pSql1 = pSql->pSubs[i]; - if (pSql1 == NULL) { - continue; - } - - SSqlRes* pRes1 = &pSql1->res; - SSqlCmd* pCmd1 = &pSql1->cmd; - - pSupporter = (SJoinSubquerySupporter*)pSql1->param; - - // wait for all subqueries completed - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd1, 0); - assert(pRes1->numOfRows >= 0 && pQueryInfo->numOfTables == 1); - - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - - if (pRes1->row >= pRes1->numOfRows) { - tscTrace("%p subquery:%p retrieve data from vnode, subquery:%d, vnodeIndex:%d", pSql, pSql1, - pSupporter->subqueryIndex, pMeterMetaInfo->vnodeIndex); - - tscResetForNextRetrieve(pRes1); - pSql1->fp = joinRetrieveCallback; - - if (pCmd1->command < TSDB_SQL_LOCAL) { - pCmd1->command = (pCmd1->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; - } - - tscProcessSql(pSql1); - } - } - - // wait for all subquery completed - tsem_wait(&pSql->rspSem); - - // update the records for each subquery - for(int32_t i = 0; i < pSql->numOfSubs; ++i) { - if (pSql->pSubs[i] == NULL) { - continue; - } - - SSqlRes* pRes1 = &pSql->pSubs[i]->res; - pRes1->numOfTotalInCurrentClause += pRes1->numOfRows; - } -} - -// all subqueries return, set the result output index -void tscSetupOutputColumnIndex(SSqlObj* pSql) { - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - tscTrace("%p all subquery response, retrieve data", pSql); - - if (pRes->pColumnIndex != NULL) { - return; // the column transfer support struct has been built - } - - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - pRes->pColumnIndex = calloc(1, sizeof(SColumnIndex) * pQueryInfo->fieldsInfo.numOfOutputCols); - - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); - - int32_t tableIndexOfSub = -1; - for (int32_t j = 0; j < pQueryInfo->numOfTables; ++j) { - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, j); - if (pMeterMetaInfo->pMeterMeta->uid == pExpr->uid) { - tableIndexOfSub = j; - break; - } - } - - assert(tableIndexOfSub >= 0 && tableIndexOfSub < pQueryInfo->numOfTables); - - SSqlCmd* pSubCmd = &pSql->pSubs[tableIndexOfSub]->cmd; - SQueryInfo* pSubQueryInfo = tscGetQueryInfoDetail(pSubCmd, 0); - - for (int32_t k = 0; k < pSubQueryInfo->exprsInfo.numOfExprs; ++k) { - SSqlExpr* pSubExpr = tscSqlExprGet(pSubQueryInfo, k); - if (pExpr->functionId == pSubExpr->functionId && pExpr->colInfo.colId == pSubExpr->colInfo.colId) { - pRes->pColumnIndex[i] = (SColumnIndex){.tableIndex = tableIndexOfSub, .columnIndex = k}; - break; - } - } - } -} - -void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { - SSqlObj* pSql = (SSqlObj*)tres; - // SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, 0, 0); - - // int32_t idx = pSql->cmd.vnodeIdx; - - // SVnodeSidList *vnodeInfo = NULL; - // if (pMeterMetaInfo->pMetricMeta != NULL) { - // vnodeInfo = tscGetVnodeSidList(pMeterMetaInfo->pMetricMeta, idx - 1); - // } - - SJoinSubquerySupporter* pSupporter = (SJoinSubquerySupporter*)param; - - // if (atomic_add_fetch_32(pSupporter->numOfComplete, 1) >= - // pSupporter->numOfTotal) { - // SSqlObj *pParentObj = pSupporter->pObj; - // - // if ((pSql->cmd.type & TSDB_QUERY_TYPE_JOIN_SEC_STAGE) != 1) { - // int32_t num = 0; - // tscFetchDatablockFromSubquery(pParentObj); - // TSKEY* ts = tscGetQualifiedTSList(pParentObj, &num); - // - // if (num <= 0) { - // // no qualified result - // } - // - // tscLaunchSecondPhaseSubqueries(pSql, ts, num); - // } else { - - // } - // } else { - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - if ((pQueryInfo->type & TSDB_QUERY_TYPE_JOIN_SEC_STAGE) != TSDB_QUERY_TYPE_JOIN_SEC_STAGE) { - if (code != TSDB_CODE_SUCCESS) { // direct call joinRetrieveCallback and set the error code - joinRetrieveCallback(param, pSql, code); - } else { // first stage query, continue to retrieve data - pSql->fp = joinRetrieveCallback; - pSql->cmd.command = TSDB_SQL_FETCH; - tscProcessSql(pSql); - } - - } else { // second stage join subquery - SSqlObj* pParentSql = pSupporter->pObj; - - if (pSupporter->pState->code != TSDB_CODE_SUCCESS) { - tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, code, - pSupporter->pState->code); - quitAllSubquery(pParentSql, pSupporter); - - return; - } - - if (code != TSDB_CODE_SUCCESS) { - tscError("%p sub query failed, code:%d, set global code:%d, index:%d", pSql, code, code, - pSupporter->subqueryIndex); - pSupporter->pState->code = code; // todo set the informative code - - quitAllSubquery(pParentSql, pSupporter); - } else { - int32_t numOfTotal = pSupporter->pState->numOfTotal; - int32_t finished = atomic_add_fetch_32(&pSupporter->pState->numOfCompleted, 1); - - if (finished >= numOfTotal) { - assert(finished == numOfTotal); - - tscSetupOutputColumnIndex(pParentSql); - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - - /** - * if the query is a continue query (vnodeIndex > 0 for projection query) for next vnode, do the retrieval of - * data instead of returning to its invoker - */ - if (pMeterMetaInfo->vnodeIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { - assert(pMeterMetaInfo->vnodeIndex < pMeterMetaInfo->pMetricMeta->numOfVnodes); - pSupporter->pState->numOfCompleted = 0; // reset the record value - - pSql->fp = joinRetrieveCallback; // continue retrieve data - pSql->cmd.command = TSDB_SQL_FETCH; - tscProcessSql(pSql); - } else { // first retrieve from vnode during the secondary stage sub-query - if (pParentSql->fp == NULL) { - tsem_wait(&pParentSql->emptyRspSem); - tsem_wait(&pParentSql->emptyRspSem); - - tsem_post(&pParentSql->rspSem); - } else { - // set the command flag must be after the semaphore been correctly set. - // pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC; - // if (pPObj->res.code == TSDB_CODE_SUCCESS) { - // (*pPObj->fp)(pPObj->param, pPObj, 0); - // } else { - // tscQueueAsyncRes(pPObj); - // } - assert(0); - } - } - } - } - } -} \ No newline at end of file diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index bc893d30049cd560016c563fc8c560e193afc89d..e864ffa7868df7fd78df9ce4644f3c99e6f43112 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -24,7 +24,7 @@ #include "qextbuffer.h" #include "tscSecondaryMerge.h" #include "tschemautil.h" -#include "tsocket.h" +#include "name.h" static void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnName, size_t valueLength); @@ -77,7 +77,7 @@ static int32_t getToStringLength(const char *pData, int32_t length, int32_t type * length((uint64_t) 123456789011) > 12, greater than sizsof(uint64_t) */ static int32_t tscMaxLengthOfTagsFields(SSqlObj *pSql) { - STableMeta *pMeta = tscGetMeterMetaInfo(&pSql->cmd, 0, 0)->pMeterMeta; + STableMeta *pMeta = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0)->pTableMeta; if (pMeta->tableType == TSDB_SUPER_TABLE || pMeta->tableType == TSDB_NORMAL_TABLE || pMeta->tableType == TSDB_STREAM_TABLE) { @@ -85,12 +85,14 @@ static int32_t tscMaxLengthOfTagsFields(SSqlObj *pSql) { } char * pTagValue = tsGetTagsValue(pMeta); - SSchema *pTagsSchema = tsGetTagSchema(pMeta); + SSchema *pTagsSchema = tscGetTableTagSchema(pMeta); int32_t len = getToStringLength(pTagValue, pTagsSchema[0].bytes, pTagsSchema[0].type); pTagValue += pTagsSchema[0].bytes; - for (int32_t i = 1; i < pMeta->numOfTags; ++i) { + int32_t numOfTags = tscGetNumOfTags(pMeta); + + for (int32_t i = 1; i < numOfTags; ++i) { int32_t tLen = getToStringLength(pTagValue, pTagsSchema[i].bytes, pTagsSchema[i].type); if (len < tLen) { len = tLen; @@ -108,8 +110,8 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { // one column for each row SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - STableMeta * pMeta = pMeterMetaInfo->pMeterMeta; + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STableMeta * pMeta = pTableMetaInfo->pTableMeta; /* * tagValueCnt is to denote the number of tags columns for meter, not metric. and is to show the column data. @@ -117,15 +119,15 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { * for metric, the value of tagValueCnt must be 0, but the numOfTags is not 0 */ - int32_t numOfRows = pMeta->numOfColumns; - int32_t totalNumOfRows = numOfRows + pMeta->numOfTags; + int32_t numOfRows = tscGetNumOfColumns(pMeta); + int32_t totalNumOfRows = numOfRows + tscGetNumOfTags(pMeta); - if (UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) { - numOfRows = pMeta->numOfColumns + pMeta->numOfTags; + if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + numOfRows = numOfRows + tscGetNumOfTags(pMeta); } tscInitResObjForLocalQuery(pSql, totalNumOfRows, rowLen); - SSchema *pSchema = tsGetSchema(pMeta); + SSchema *pSchema = tscGetTableSchema(pMeta); for (int32_t i = 0; i < numOfRows; ++i) { TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, 0); @@ -146,13 +148,13 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { *(int32_t *)(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 2) * totalNumOfRows + pField->bytes * i) = bytes; pField = tscFieldInfoGetField(pQueryInfo, 3); - if (i >= pMeta->numOfColumns && pMeta->numOfTags != 0) { + if (i >= tscGetNumOfColumns(pMeta) && tscGetNumOfTags(pMeta) != 0) { strncpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i, "tag", strlen("tag") + 1); } } - if (UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) { + if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { return 0; } @@ -265,7 +267,7 @@ static int32_t tscBuildMeterSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, static int32_t tscProcessDescribeTable(SSqlObj *pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - assert(tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0)->pMeterMeta != NULL); + assert(tscGetMetaInfo(pQueryInfo, 0)->pTableMeta != NULL); const int32_t NUM_OF_DESCRIBE_TABLE_COLUMNS = 4; const int32_t TYPE_COLUMN_LENGTH = 16; @@ -290,15 +292,15 @@ static int tscBuildMetricTagProjectionResult(SSqlObj *pSql) { SSqlRes * pRes = &pSql->res; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - SSuperTableMeta *pMetricMeta = pMeterMetaInfo->pMetricMeta; - SSchema * pSchema = tsGetTagSchema(pMeterMetaInfo->pMeterMeta); + SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta; + SSchema * pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); int32_t vOffset[TSDB_MAX_COLUMNS] = {0}; - for (int32_t f = 1; f < pMeterMetaInfo->numOfTags; ++f) { - int16_t tagColumnIndex = pMeterMetaInfo->tagColumnIndex[f - 1]; + for (int32_t f = 1; f < pTableMetaInfo->numOfTags; ++f) { + int16_t tagColumnIndex = pTableMetaInfo->tagColumnIndex[f - 1]; if (tagColumnIndex == -1) { vOffset[f] = vOffset[f - 1] + TSDB_TABLE_NAME_LEN; } else { @@ -343,7 +345,7 @@ static int tscBuildMetricTagSqlFunctionResult(SSqlObj *pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - SSuperTableMeta *pMetricMeta = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0)->pMetricMeta; + SSuperTableMeta *pMetricMeta = tscGetMetaInfo(pQueryInfo, 0)->pMetricMeta; int32_t totalNumOfResults = 1; // count function only produce one result int32_t rowLen = tscGetResRowLength(pQueryInfo); @@ -375,8 +377,8 @@ static int tscProcessQueryTags(SSqlObj *pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - STableMeta *pMeterMeta = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0)->pMeterMeta; - if (pMeterMeta == NULL || pMeterMeta->numOfTags == 0 || pMeterMeta->numOfColumns == 0) { + STableMeta *pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; + if (pTableMeta == NULL || tscGetNumOfTags(pTableMeta) == 0 || tscGetNumOfColumns(pTableMeta) == 0) { strcpy(pCmd->payload, "invalid table"); pSql->res.code = TSDB_CODE_INVALID_TABLE; return pSql->res.code; diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 6c26437b80caefe2a79a522e401098f9edda6124..b96beb35fd654301203f0595465f6dbb685aab9c 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -28,6 +28,7 @@ #include "taosdef.h" #include "tlog.h" +#include "tscSubquery.h" #include "tstoken.h" #include "ttime.h" @@ -496,15 +497,17 @@ static int32_t rowDataCompar(const void *lhs, const void *rhs) { } } -int tsParseValues(char **str, STableDataBlocks *pDataBlock, STableMeta *pMeterMeta, int maxRows, +int tsParseValues(char **str, STableDataBlocks *pDataBlock, STableMeta *pTableMeta, int maxRows, SParsedDataColInfo *spd, char *error, int32_t *code, char *tmpTokenBuf) { int32_t index = 0; SSQLToken sToken; int16_t numOfRows = 0; - SSchema *pSchema = tsGetSchema(pMeterMeta); - int32_t precision = pMeterMeta->precision; + SSchema *pSchema = tscGetTableSchema(pTableMeta); + STableComInfo tinfo = tscGetTableInfo(pTableMeta); + + int32_t precision = tinfo.precision; if (spd->hasVal[0] == false) { strcpy(error, "primary timestamp column can not be null"); @@ -518,9 +521,9 @@ int tsParseValues(char **str, STableDataBlocks *pDataBlock, STableMeta *pMeterMe if (sToken.n == 0 || sToken.type != TK_LP) break; *str += index; - if (numOfRows >= maxRows || pDataBlock->size + pMeterMeta->rowSize >= pDataBlock->nAllocSize) { + if (numOfRows >= maxRows || pDataBlock->size + tinfo.rowSize >= pDataBlock->nAllocSize) { int32_t tSize; - int32_t retcode = tscAllocateMemIfNeed(pDataBlock, pMeterMeta->rowSize, &tSize); + int32_t retcode = tscAllocateMemIfNeed(pDataBlock, tinfo.rowSize, &tSize); if (retcode != TSDB_CODE_SUCCESS) { //TODO pass the correct error code to client strcpy(error, "client out of memory"); *code = retcode; @@ -601,10 +604,10 @@ int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int3 return TSDB_CODE_SUCCESS; } -static void tsSetBlockInfo(SShellSubmitBlock *pBlocks, const STableMeta *pMeterMeta, int32_t numOfRows) { - pBlocks->sid = pMeterMeta->sid; - pBlocks->uid = pMeterMeta->uid; - pBlocks->sversion = pMeterMeta->sversion; +static void tsSetBlockInfo(SShellSubmitBlock *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) { + pBlocks->sid = pTableMeta->sid; + pBlocks->uid = pTableMeta->uid; + pBlocks->sversion = pTableMeta->sversion; pBlocks->numOfRows += numOfRows; } @@ -654,19 +657,20 @@ void sortRemoveDuplicates(STableDataBlocks *dataBuf) { static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableHashList, char **str, SParsedDataColInfo *spd, int32_t *totalNum) { SSqlCmd * pCmd = &pSql->cmd; - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); - STableMeta * pMeterMeta = pMeterMetaInfo->pMeterMeta; - + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; + STableComInfo tinfo = tscGetTableInfo(pTableMeta); + STableDataBlocks *dataBuf = NULL; - int32_t ret = tscGetDataBlockFromList(pTableHashList, pCmd->pDataBlocks, pMeterMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SShellSubmitBlock), pMeterMeta->rowSize, pMeterMetaInfo->name, - pMeterMeta, &dataBuf); + int32_t ret = tscGetDataBlockFromList(pTableHashList, pCmd->pDataBlocks, pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, + sizeof(SShellSubmitBlock), tinfo.rowSize, pTableMetaInfo->name, + pTableMeta, &dataBuf); if (ret != TSDB_CODE_SUCCESS) { return ret; } int32_t maxNumOfRows; - ret = tscAllocateMemIfNeed(dataBuf, pMeterMeta->rowSize, &maxNumOfRows); + ret = tscAllocateMemIfNeed(dataBuf, tinfo.rowSize, &maxNumOfRows); if (TSDB_CODE_SUCCESS != ret) { return TSDB_CODE_CLI_OUT_OF_MEMORY; } @@ -677,7 +681,7 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableHashList, char return TSDB_CODE_CLI_OUT_OF_MEMORY; } - int32_t numOfRows = tsParseValues(str, dataBuf, pMeterMeta, maxNumOfRows, spd, pCmd->payload, &code, tmpTokenBuf); + int32_t numOfRows = tsParseValues(str, dataBuf, pTableMeta, maxNumOfRows, spd, pCmd->payload, &code, tmpTokenBuf); free(tmpTokenBuf); if (numOfRows <= 0) { return code; @@ -692,9 +696,9 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableHashList, char } SShellSubmitBlock *pBlocks = (SShellSubmitBlock *)(dataBuf->pData); - tsSetBlockInfo(pBlocks, pMeterMeta, numOfRows); + tsSetBlockInfo(pBlocks, pTableMeta, numOfRows); - dataBuf->vgid = pMeterMeta->vgid; + dataBuf->vgid = pTableMeta->vgid; dataBuf->numOfTables = 1; /* @@ -756,7 +760,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return TSDB_CODE_INVALID_SQL; } - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, TABLE_INDEX); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, TABLE_INDEX); if (sToken.type == TK_USING) { // create table if not exists according to the super table index = 0; @@ -767,34 +771,35 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { memset(pTag, 0, sizeof(STagData)); /* - * the source super table is moved to the secondary position of the pMeterMetaInfo list + * the source super table is moved to the secondary position of the pTableMetaInfo list */ if (pQueryInfo->numOfTables < 2) { - tscAddEmptyMeterMetaInfo(pQueryInfo); + tscAddEmptyMetaInfo(pQueryInfo); } - SMeterMetaInfo *pSTableMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, STABLE_INDEX); + STableMetaInfo *pSTableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX); setMeterID(pSTableMeterMetaInfo, &sToken, pSql); strncpy(pTag->name, pSTableMeterMetaInfo->name, TSDB_TABLE_ID_LEN); - code = tscGetMeterMeta(pSql, pSTableMeterMetaInfo); + code = tscGetTableMeta(pSql, pSTableMeterMetaInfo); if (code != TSDB_CODE_SUCCESS) { return code; } - if (!UTIL_METER_IS_SUPERTABLE(pSTableMeterMetaInfo)) { + if (!UTIL_TABLE_IS_SUPERTABLE(pSTableMeterMetaInfo)) { return tscInvalidSQLErrMsg(pCmd->payload, "create table only from super table is allowed", sToken.z); } - SSchema *pTagSchema = tsGetTagSchema(pSTableMeterMetaInfo->pMeterMeta); - + SSchema *pTagSchema = tscGetTableTagSchema(pSTableMeterMetaInfo->pTableMeta); + STableComInfo tinfo = tscGetTableInfo(pSTableMeterMetaInfo->pTableMeta); + index = 0; sToken = tStrGetToken(sql, &index, false, 0, NULL); sql += index; SParsedDataColInfo spd = {0}; - - uint8_t numOfTags = pSTableMeterMetaInfo->pMeterMeta->numOfTags; + + uint8_t numOfTags = tscGetNumOfTags(pSTableMeterMetaInfo->pTableMeta); spd.numOfCols = numOfTags; // if specify some tags column @@ -881,8 +886,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { sToken.n -= 2; } - code = tsParseOneColumnData(&pTagSchema[colIndex], &sToken, tagVal, pCmd->payload, &sql, false, - pSTableMeterMetaInfo->pMeterMeta->precision); + code = tsParseOneColumnData(&pTagSchema[colIndex], &sToken, tagVal, pCmd->payload, &sql, false, tinfo.precision); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -917,13 +921,13 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return tscInvalidSQLErrMsg(pCmd->payload, "invalid table name", *sqlstr); } - int32_t ret = setMeterID(pMeterMetaInfo, &tableToken, pSql); + int32_t ret = setMeterID(pTableMetaInfo, &tableToken, pSql); if (ret != TSDB_CODE_SUCCESS) { return ret; } createTable = true; - code = tscGetMeterMetaEx(pSql, pMeterMetaInfo, true); + code = tscGetMeterMetaEx(pSql, pTableMetaInfo, true); if (TSDB_CODE_ACTION_IN_PROGRESS == code) { return code; } @@ -934,7 +938,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { } else { sql = sToken.z; } - code = tscGetMeterMeta(pSql, pMeterMetaInfo); + code = tscGetTableMeta(pSql, pTableMetaInfo); } int32_t len = cend - cstart + 1; @@ -983,15 +987,15 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { int32_t totalNum = 0; int32_t code = TSDB_CODE_SUCCESS; - SMeterMetaInfo *pMeterMetaInfo = NULL; + STableMetaInfo *pTableMetaInfo = NULL; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); assert(pQueryInfo != NULL); if (pQueryInfo->numOfTables == 0) { - pMeterMetaInfo = tscAddEmptyMeterMetaInfo(pQueryInfo); + pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); } else { - pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); } if ((code = tscAllocPayload(pCmd, TSDB_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { @@ -1050,7 +1054,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { goto _error_clean; } - if ((code = setMeterID(pMeterMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) { + if ((code = setMeterID(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) { goto _error_clean; } @@ -1078,7 +1082,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { goto _error_clean; // TODO: should _clean or _error_clean to async flow ???? } - if (UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) { + if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { code = tscInvalidSQLErrMsg(pCmd->payload, "insert data into super table is not supported", NULL); goto _error_clean; } @@ -1091,12 +1095,16 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { code = tscInvalidSQLErrMsg(pCmd->payload, "keyword VALUES or FILE required", sToken.z); goto _error_clean; } - + + int32_t numOfCols = tscGetNumOfTags(pTableMetaInfo->pTableMeta); + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); + if (sToken.type == TK_VALUES) { - SParsedDataColInfo spd = {.numOfCols = pMeterMetaInfo->pMeterMeta->numOfColumns}; - SSchema * pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); + SParsedDataColInfo spd = {.numOfCols = numOfCols}; + + SSchema *pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - tscSetAssignedColumnInfo(&spd, pSchema, pMeterMetaInfo->pMeterMeta->numOfColumns); + tscSetAssignedColumnInfo(&spd, pSchema, numOfCols); if (validateDataSource(pCmd, DATA_FROM_SQL_STRING, sToken.z) != TSDB_CODE_SUCCESS) { goto _error_clean; @@ -1136,10 +1144,10 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { wordfree(&full_path); STableDataBlocks *pDataBlock = NULL; - STableMeta* pMeterMeta = pMeterMetaInfo->pMeterMeta; + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - int32_t ret = tscCreateDataBlock(PATH_MAX, pMeterMeta->rowSize, sizeof(SShellSubmitBlock), pMeterMetaInfo->name, - pMeterMeta, &pDataBlock); + int32_t ret = tscCreateDataBlock(PATH_MAX, tinfo.rowSize, sizeof(SShellSubmitBlock), pTableMetaInfo->name, + pTableMeta, &pDataBlock); if (ret != TSDB_CODE_SUCCESS) { goto _error_clean; } @@ -1148,18 +1156,18 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { strcpy(pDataBlock->filename, fname); } else if (sToken.type == TK_LP) { /* insert into tablename(col1, col2,..., coln) values(v1, v2,... vn); */ - STableMeta *pMeterMeta = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0)->pMeterMeta; - SSchema * pSchema = tsGetSchema(pMeterMeta); + STableMeta *pTableMeta = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0)->pTableMeta; + SSchema * pSchema = tscGetTableSchema(pTableMeta); if (validateDataSource(pCmd, DATA_FROM_SQL_STRING, sToken.z) != TSDB_CODE_SUCCESS) { goto _error_clean; } SParsedDataColInfo spd = {0}; - spd.numOfCols = pMeterMeta->numOfColumns; + spd.numOfCols = tinfo.numOfColumns; int16_t offset[TSDB_MAX_COLUMNS] = {0}; - for (int32_t t = 1; t < pMeterMeta->numOfColumns; ++t) { + for (int32_t t = 1; t < tinfo.numOfColumns; ++t) { offset[t] = offset[t - 1] + pSchema[t - 1].bytes; } @@ -1181,7 +1189,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { bool findColumnIndex = false; // todo speedup by using hash list - for (int32_t t = 0; t < pMeterMeta->numOfColumns; ++t) { + for (int32_t t = 0; t < tinfo.numOfColumns; ++t) { if (strncmp(sToken.z, pSchema[t].name, sToken.n) == 0 && strlen(pSchema[t].name) == sToken.n) { SParsedColElem *pElem = &spd.elems[spd.numOfAssignedCols++]; pElem->offset = offset[t]; @@ -1204,7 +1212,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { } } - if (spd.numOfAssignedCols == 0 || spd.numOfAssignedCols > pMeterMeta->numOfColumns) { + if (spd.numOfAssignedCols == 0 || spd.numOfAssignedCols > tinfo.numOfColumns) { code = tscInvalidSQLErrMsg(pCmd->payload, "column name expected", sToken.z); goto _error_clean; } @@ -1245,10 +1253,10 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { goto _error_clean; } - pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, 0, 0); + pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); // set the next sent data vnode index in data block arraylist - pMeterMetaInfo->vnodeIndex = 1; + pTableMetaInfo->vnodeIndex = 1; } else { pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); } @@ -1317,7 +1325,7 @@ int tsParseSql(SSqlObj *pSql, bool multiVnodeInsertion) { pSql->fetchFp = pSql->fp; // replace user defined callback function with multi-insert proxy function - pSql->fp = (void(*)())launchMultivnodeInsert; + pSql->fp = (void(*)())tscHandleMultivnodeInsert; } ret = tsParseInsertSql(pSql); @@ -1333,7 +1341,7 @@ int tsParseSql(SSqlObj *pSql, bool multiVnodeInsertion) { } /* - * the pRes->code may be modified or even released by another thread in tscMeterMetaCallBack + * the pRes->code may be modified or even released by another thread in tscTableMetaCallBack * function, so do NOT use pRes->code to determine if the getMeterMeta/getMetricMeta function * invokes new threads to get data from mnode or simply retrieves data from cache. * @@ -1348,10 +1356,10 @@ static int doPackSendDataBlock(SSqlObj *pSql, int32_t numOfRows, STableDataBlock SSqlCmd *pCmd = &pSql->cmd; assert(pCmd->numOfClause == 1); - STableMeta *pMeterMeta = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0)->pMeterMeta; + STableMeta *pTableMeta = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0)->pTableMeta; SShellSubmitBlock *pBlocks = (SShellSubmitBlock *)(pTableDataBlocks->pData); - tsSetBlockInfo(pBlocks, pMeterMeta, numOfRows); + tsSetBlockInfo(pBlocks, pTableMeta, numOfRows); if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) { return code; @@ -1381,16 +1389,18 @@ static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp, char *tmpTokenBuf) { int32_t code = 0; int nrows = 0; - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); - STableMeta * pMeterMeta = pMeterMetaInfo->pMeterMeta; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; + STableComInfo tinfo = tscGetTableInfo(pTableMeta); + assert(pCmd->numOfClause == 1); - int32_t rowSize = pMeterMeta->rowSize; + int32_t rowSize = tinfo.rowSize; pCmd->pDataBlocks = tscCreateBlockArrayList(); STableDataBlocks *pTableDataBlock = NULL; int32_t ret = tscCreateDataBlock(TSDB_PAYLOAD_SIZE, rowSize, sizeof(SShellSubmitBlock), - pMeterMetaInfo->name, pMeterMeta, &pTableDataBlock); + pTableMetaInfo->name, pTableMeta, &pTableDataBlock); if (ret != TSDB_CODE_SUCCESS) { return -1; } @@ -1401,10 +1411,10 @@ static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp, char *tmpTokenBuf) { if (TSDB_CODE_SUCCESS != code) return -1; int count = 0; - SParsedDataColInfo spd = {.numOfCols = pMeterMeta->numOfColumns}; - SSchema * pSchema = tsGetSchema(pMeterMeta); + SParsedDataColInfo spd = {.numOfCols = tinfo.numOfColumns}; + SSchema * pSchema = tscGetTableSchema(pTableMeta); - tscSetAssignedColumnInfo(&spd, pSchema, pMeterMeta->numOfColumns); + tscSetAssignedColumnInfo(&spd, pSchema, tinfo.numOfColumns); while ((readLen = getline(&line, &n, fp)) != -1) { // line[--readLen] = '\0'; @@ -1414,7 +1424,7 @@ static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp, char *tmpTokenBuf) { char *lineptr = line; strtolower(line, line); - len = tsParseOneRowData(&lineptr, pTableDataBlock, pSchema, &spd, pCmd->payload, pMeterMeta->precision, &code, tmpTokenBuf); + len = tsParseOneRowData(&lineptr, pTableDataBlock, pSchema, &spd, pCmd->payload, tinfo.precision, &code, tmpTokenBuf); if (len <= 0 || pTableDataBlock->numOfParams > 0) { pSql->res.code = code; return (-code); @@ -1431,7 +1441,7 @@ static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp, char *tmpTokenBuf) { pTableDataBlock = pCmd->pDataBlocks->pData[0]; pTableDataBlock->size = sizeof(SShellSubmitBlock); - pTableDataBlock->rowSize = pMeterMeta->rowSize; + pTableDataBlock->rowSize = tinfo.rowSize; numOfRows += pSql->res.numOfRows; pSql->res.numOfRows = 0; @@ -1473,25 +1483,25 @@ void tscProcessMultiVnodesInsert(SSqlObj *pSql) { } STableDataBlocks *pDataBlock = NULL; - SMeterMetaInfo * pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); + STableMetaInfo * pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); assert(pCmd->numOfClause == 1); int32_t code = TSDB_CODE_SUCCESS; /* the first block has been sent to server in processSQL function */ - assert(pMeterMetaInfo->vnodeIndex >= 1 && pCmd->pDataBlocks != NULL); + assert(pTableMetaInfo->vnodeIndex >= 1 && pCmd->pDataBlocks != NULL); - if (pMeterMetaInfo->vnodeIndex < pCmd->pDataBlocks->nSize) { + if (pTableMetaInfo->vnodeIndex < pCmd->pDataBlocks->nSize) { SDataBlockList *pDataBlocks = pCmd->pDataBlocks; - for (int32_t i = pMeterMetaInfo->vnodeIndex; i < pDataBlocks->nSize; ++i) { + for (int32_t i = pTableMetaInfo->vnodeIndex; i < pDataBlocks->nSize; ++i) { pDataBlock = pDataBlocks->pData[i]; if (pDataBlock == NULL) { continue; } if ((code = tscCopyDataBlockToPayload(pSql, pDataBlock)) != TSDB_CODE_SUCCESS) { - tscTrace("%p build submit data block failed, vnodeIdx:%d, total:%d", pSql, pMeterMetaInfo->vnodeIndex, + tscTrace("%p build submit data block failed, vnodeIdx:%d, total:%d", pSql, pTableMetaInfo->vnodeIndex, pDataBlocks->nSize); continue; } @@ -1512,7 +1522,7 @@ void tscProcessMultiVnodesInsertFromFile(SSqlObj *pSql) { } SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableDataBlocks *pDataBlock = NULL; int32_t affected_rows = 0; @@ -1543,10 +1553,10 @@ void tscProcessMultiVnodesInsertFromFile(SSqlObj *pSql) { continue; } - strncpy(pMeterMetaInfo->name, pDataBlock->tableId, TSDB_TABLE_ID_LEN); + strncpy(pTableMetaInfo->name, pDataBlock->tableId, TSDB_TABLE_ID_LEN); memset(pDataBlock->pData, 0, pDataBlock->nAllocSize); - int32_t ret = tscGetMeterMeta(pSql, pMeterMetaInfo); + int32_t ret = tscGetTableMeta(pSql, pTableMetaInfo); if (ret != TSDB_CODE_SUCCESS) { tscError("%p get meter meta failed, abort", pSql); continue; diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index a20d7cad1c983ceb54ac3c7c66f19866143d1840..96215ce73c269c1a2ec0d99bb0d7a6a217b97396 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -407,8 +407,8 @@ static int insertStmtReset(STscStmt* pStmt) { } pCmd->batchSize = 0; - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); - pMeterMetaInfo->vnodeIndex = 0; + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + pTableMetaInfo->vnodeIndex = 0; return TSDB_CODE_SUCCESS; } @@ -421,7 +421,7 @@ static int insertStmtExecute(STscStmt* stmt) { ++pCmd->batchSize; } - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); assert(pCmd->numOfClause == 1); if (pCmd->pDataBlocks->nSize > 0) { @@ -438,7 +438,7 @@ static int insertStmtExecute(STscStmt* stmt) { } // set the next sent data vnode index in data block arraylist - pMeterMetaInfo->vnodeIndex = 1; + pTableMetaInfo->vnodeIndex = 1; } else { pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index e814965b13d586e11fce309490c15efaddc0428f..e0850a7139f72a7569560c8131d5c8c2ff69fa14 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -29,6 +29,8 @@ #include "tsclient.h" #include "ttokendef.h" +#include "name.h" + #define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0" // -1 is tbname column index, so here use the -2 as the initial value @@ -149,10 +151,12 @@ static int setColumnFilterInfoForTimestamp(SQueryInfo* pQueryInfo, tVariant* pVa strdequote(pVar->pz); char* seg = strnchr(pVar->pz, '-', pVar->nLen, false); - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); + if (seg != NULL) { - if (taosParseTime(pVar->pz, &time, pVar->nLen, pMeterMetaInfo->pMeterMeta->precision) != TSDB_CODE_SUCCESS) { + if (taosParseTime(pVar->pz, &time, pVar->nLen, tinfo.precision) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(pQueryInfo->msg, msg); } } else { @@ -207,7 +211,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { int32_t code = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo); assert(pQueryInfo->numOfTables == 0); - SMeterMetaInfo* pMeterMetaInfo = tscAddEmptyMeterMetaInfo(pQueryInfo); + STableMetaInfo* pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); pCmd->command = pInfo->type; @@ -229,7 +233,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (pInfo->type == TSDB_SQL_DROP_DB) { assert(pInfo->pDCLInfo->nTokens == 1); - code = setObjFullName(pMeterMetaInfo->name, getAccountId(pSql), pzName, NULL, NULL); + code = setObjFullName(pTableMetaInfo->name, getAccountId(pSql), pzName, NULL, NULL); if (code != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -237,7 +241,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } else if (pInfo->type == TSDB_SQL_DROP_TABLE) { assert(pInfo->pDCLInfo->nTokens == 1); - if (setMeterID(pMeterMetaInfo, pzName, pSql) != TSDB_CODE_SUCCESS) { + if (setMeterID(pTableMetaInfo, pzName, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } } else if (pInfo->type == TSDB_SQL_DROP_DNODE) { @@ -245,13 +249,13 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - strncpy(pMeterMetaInfo->name, pzName->z, pzName->n); + strncpy(pTableMetaInfo->name, pzName->z, pzName->n); } else { // drop user if (pzName->n > TSDB_USER_LEN) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } - strncpy(pMeterMetaInfo->name, pzName->z, pzName->n); + strncpy(pTableMetaInfo->name, pzName->z, pzName->n); } break; @@ -265,7 +269,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } - int32_t ret = setObjFullName(pMeterMetaInfo->name, getAccountId(pSql), pToken, NULL, NULL); + int32_t ret = setObjFullName(pTableMetaInfo->name, getAccountId(pSql), pToken, NULL, NULL); if (ret != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } @@ -295,7 +299,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - int32_t ret = setObjFullName(pMeterMetaInfo->name, getAccountId(pSql), &(pCreateDB->dbname), NULL, NULL); + int32_t ret = setObjFullName(pTableMetaInfo->name, getAccountId(pSql), &(pCreateDB->dbname), NULL, NULL); if (ret != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -370,11 +374,11 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - if (setMeterID(pMeterMetaInfo, pToken, pSql) != TSDB_CODE_SUCCESS) { + if (setMeterID(pTableMetaInfo, pToken, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - return tscGetMeterMeta(pSql, pMeterMetaInfo); + return tscGetTableMeta(pSql, pTableMetaInfo); } case TSDB_SQL_CFG_DNODE: { @@ -585,8 +589,9 @@ int32_t parseIntervalClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { const char* msg1 = "invalid query expression"; const char* msg2 = "interval cannot be less than 10 ms"; - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); + if (pQuerySql->interval.type == 0 || pQuerySql->interval.n == 0) { return TSDB_CODE_SUCCESS; } @@ -598,12 +603,12 @@ int32_t parseIntervalClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { } // if the unit of time window value is millisecond, change the value from microsecond - if (pMeterMetaInfo->pMeterMeta->precision == TSDB_TIME_PRECISION_MILLI) { + if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { pQueryInfo->intervalTime = pQueryInfo->intervalTime / 1000; } /* parser has filter the illegal type, no need to check here */ - pQueryInfo->intervalTimeUnit = pQuerySql->interval.z[pQuerySql->interval.n - 1]; + pQueryInfo->slidingTimeUnit = pQuerySql->interval.z[pQuerySql->interval.n - 1]; // interval cannot be less than 10 milliseconds if (pQueryInfo->intervalTime < tsMinIntervalTime) { @@ -643,8 +648,8 @@ int32_t parseIntervalClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL; for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, i); - if (pMeterMetaInfo->pMeterMeta->uid == uid) { + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); + if (pTableMetaInfo->pTableMeta->uid == uid) { tableIndex = i; break; } @@ -676,12 +681,13 @@ int32_t parseSlidingClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { const char* msg0 = "sliding value too small"; const char* msg1 = "sliding value no larger than the interval value"; - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); SSQLToken* pSliding = &pQuerySql->sliding; + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); if (pSliding->n != 0) { getTimestampInUsFromStr(pSliding->z, pSliding->n, &pQueryInfo->slidingTime); - if (pMeterMetaInfo->pMeterMeta->precision == TSDB_TIME_PRECISION_MILLI) { + if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { pQueryInfo->slidingTime /= 1000; } @@ -699,27 +705,27 @@ int32_t parseSlidingClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { return TSDB_CODE_SUCCESS; } -int32_t setMeterID(SMeterMetaInfo* pMeterMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql) { +int32_t setMeterID(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql) { const char* msg = "name too long"; SSqlCmd* pCmd = &pSql->cmd; int32_t code = TSDB_CODE_SUCCESS; - // backup the old name in pMeterMetaInfo - size_t size = strlen(pMeterMetaInfo->name); + // backup the old name in pTableMetaInfo + size_t size = strlen(pTableMetaInfo->name); char* oldName = NULL; if (size > 0) { - oldName = strdup(pMeterMetaInfo->name); + oldName = strdup(pTableMetaInfo->name); } if (hasSpecifyDB(pzTableName)) { // db has been specified in sql string so we ignore current db path - code = setObjFullName(pMeterMetaInfo->name, getAccountId(pSql), NULL, pzTableName, NULL); + code = setObjFullName(pTableMetaInfo->name, getAccountId(pSql), NULL, pzTableName, NULL); } else { // get current DB name first, then set it into path SSQLToken t = {0}; getCurrentDBName(pSql, &t); - code = setObjFullName(pMeterMetaInfo->name, NULL, &t, pzTableName, NULL); + code = setObjFullName(pTableMetaInfo->name, NULL, &t, pzTableName, NULL); } if (code != TSDB_CODE_SUCCESS) { @@ -736,11 +742,11 @@ int32_t setMeterID(SMeterMetaInfo* pMeterMetaInfo, SSQLToken* pzTableName, SSqlO * that are corresponding to the old name for the new table name. */ if (size > 0) { - if (strncasecmp(oldName, pMeterMetaInfo->name, tListLen(pMeterMetaInfo->name)) != 0) { - tscClearMeterMetaInfo(pMeterMetaInfo, false); + if (strncasecmp(oldName, pTableMetaInfo->name, tListLen(pTableMetaInfo->name)) != 0) { + tscClearMeterMetaInfo(pTableMetaInfo, false); } } else { - assert(pMeterMetaInfo->pMeterMeta == NULL && pMeterMetaInfo->pMetricMeta == NULL); + assert(pTableMetaInfo->pTableMeta == NULL && pTableMetaInfo->pMetricMeta == NULL); } tfree(oldName); @@ -890,11 +896,14 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { assert(pCmd->numOfClause == 1); - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); - STableMeta* pMeterMeta = pMeterMetaInfo->pMeterMeta; + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + int32_t numOfTags = tscGetNumOfTags(pTableMeta); + int32_t numOfCols = tscGetNumOfColumns(pTableMeta); + // no more than 6 tags - if (pMeterMeta->numOfTags == TSDB_MAX_TAGS) { + if (numOfTags == TSDB_MAX_TAGS) { char msg[128] = {0}; sprintf(msg, "tags no more than %d", TSDB_MAX_TAGS); @@ -913,10 +922,10 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { return false; } - SSchema* pTagSchema = tsGetTagSchema(pMeterMetaInfo->pMeterMeta); + SSchema* pTagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); int32_t nLen = 0; - for (int32_t i = 0; i < pMeterMeta->numOfTags; ++i) { + for (int32_t i = 0; i < numOfTags; ++i) { nLen += pTagSchema[i].bytes; } @@ -939,9 +948,9 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { } // field name must be unique - SSchema* pSchema = tsGetSchema(pMeterMeta); + SSchema* pSchema = tscGetTableSchema(pTableMeta); - for (int32_t i = 0; i < pMeterMeta->numOfTags + pMeterMeta->numOfColumns; ++i) { + for (int32_t i = 0; i < numOfTags + numOfCols; ++i) { if (strncasecmp(pTagField->name, pSchema[i].name, TSDB_COL_NAME_LEN) == 0) { invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); return false; @@ -960,12 +969,14 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { const char* msg6 = "invalid column length"; assert(pCmd->numOfClause == 1); - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); - STableMeta* pMeterMeta = pMeterMetaInfo->pMeterMeta; - + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + + int32_t numOfTags = tscGetNumOfTags(pTableMeta); + int32_t numOfCols = tscGetNumOfColumns(pTableMeta); + // no more max columns - if (pMeterMeta->numOfColumns >= TSDB_MAX_COLUMNS || - pMeterMeta->numOfTags + pMeterMeta->numOfColumns >= TSDB_MAX_COLUMNS) { + if (numOfCols >= TSDB_MAX_COLUMNS || numOfTags + numOfCols >= TSDB_MAX_COLUMNS) { invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); return false; } @@ -980,10 +991,10 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { return false; } - SSchema* pSchema = tsGetSchema(pMeterMeta); + SSchema* pSchema = tscGetTableSchema(pTableMeta); int32_t nLen = 0; - for (int32_t i = 0; i < pMeterMeta->numOfColumns; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { nLen += pSchema[i].bytes; } @@ -999,7 +1010,7 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { } // field name must be unique - for (int32_t i = 0; i < pMeterMeta->numOfTags + pMeterMeta->numOfColumns; ++i) { + for (int32_t i = 0; i < numOfTags + numOfCols; ++i) { if (strncasecmp(pColField->name, pSchema[i].name, TSDB_COL_NAME_LEN) == 0) { invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); return false; @@ -1190,7 +1201,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel pFuncExpr->interResBytes = sizeof(double); pFuncExpr->resType = TSDB_DATA_TYPE_DOUBLE; - SSqlBinaryExprInfo* pBinExprInfo = &pFuncExpr->pBinExprInfo; + SSqlBinaryExprInfo* pBinExprInfo = &pFuncExpr->binExprInfo; tSQLSyntaxNode* pNode = NULL; SColIndexEx* pColIndex = NULL; @@ -1238,10 +1249,11 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel if (isSTable) { pQueryInfo->type |= TSDB_QUERY_TYPE_STABLE_QUERY; - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); + if (tscQueryMetricTags(pQueryInfo)) { // local handle the metric tag query - pCmd->count = pMeterMetaInfo->pMeterMeta->numOfColumns; // the number of meter schema, tricky. + pCmd->count = numOfCols; // the number of meter schema, tricky. pQueryInfo->command = TSDB_SQL_RETRIEVE_TAGS; } @@ -1272,11 +1284,11 @@ int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnLi } SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t colIdx, int32_t tableIndex) { - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, tableIndex); - STableMeta* pMeterMeta = pMeterMetaInfo->pMeterMeta; - - SSchema* pSchema = tsGetColumnSchema(pMeterMeta, colIdx); - int32_t numOfCols = pMeterMeta->numOfColumns; + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + int32_t numOfCols = tscGetNumOfColumns(pTableMeta); + + SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, colIdx); int16_t functionId = (int16_t)((colIdx >= numOfCols) ? TSDB_FUNC_TAGPRJ : TSDB_FUNC_PRJ); @@ -1295,39 +1307,39 @@ SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t c } void addRequiredTagColumn(SQueryInfo* pQueryInfo, int32_t tagColIndex, int32_t tableIndex) { - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, tableIndex); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); - if (pMeterMetaInfo->numOfTags == 0 || pMeterMetaInfo->tagColumnIndex[pMeterMetaInfo->numOfTags - 1] < tagColIndex) { - pMeterMetaInfo->tagColumnIndex[pMeterMetaInfo->numOfTags++] = tagColIndex; + if (pTableMetaInfo->numOfTags == 0 || pTableMetaInfo->tagColumnIndex[pTableMetaInfo->numOfTags - 1] < tagColIndex) { + pTableMetaInfo->tagColumnIndex[pTableMetaInfo->numOfTags++] = tagColIndex; } else { // find the appropriate position - for (int32_t i = 0; i < pMeterMetaInfo->numOfTags; ++i) { - if (tagColIndex > pMeterMetaInfo->tagColumnIndex[i]) { + for (int32_t i = 0; i < pTableMetaInfo->numOfTags; ++i) { + if (tagColIndex > pTableMetaInfo->tagColumnIndex[i]) { continue; - } else if (tagColIndex == pMeterMetaInfo->tagColumnIndex[i]) { + } else if (tagColIndex == pTableMetaInfo->tagColumnIndex[i]) { break; } else { - memmove(&pMeterMetaInfo->tagColumnIndex[i + 1], &pMeterMetaInfo->tagColumnIndex[i], - sizeof(pMeterMetaInfo->tagColumnIndex[0]) * (pMeterMetaInfo->numOfTags - i)); + memmove(&pTableMetaInfo->tagColumnIndex[i + 1], &pTableMetaInfo->tagColumnIndex[i], + sizeof(pTableMetaInfo->tagColumnIndex[0]) * (pTableMetaInfo->numOfTags - i)); - pMeterMetaInfo->tagColumnIndex[i] = tagColIndex; + pTableMetaInfo->tagColumnIndex[i] = tagColIndex; - pMeterMetaInfo->numOfTags++; + pTableMetaInfo->numOfTags++; break; } } } // plus one means tbname - assert(tagColIndex >= -1 && tagColIndex < TSDB_MAX_TAGS && pMeterMetaInfo->numOfTags <= TSDB_MAX_TAGS + 1); + assert(tagColIndex >= -1 && tagColIndex < TSDB_MAX_TAGS && pTableMetaInfo->numOfTags <= TSDB_MAX_TAGS + 1); } static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSQLExprItem* pItem) { SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, startPos, pIndex->columnIndex, pIndex->tableIndex); - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, pIndex->tableIndex); - STableMeta* pMeterMeta = pMeterMetaInfo->pMeterMeta; - - SSchema* pSchema = tsGetColumnSchema(pMeterMeta, pIndex->columnIndex); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + + SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex); char* colName = (pItem->aliasName == NULL) ? pSchema->name : pItem->aliasName; strncpy(pExpr->aliasName, colName, tListLen(pExpr->aliasName)); @@ -1336,7 +1348,7 @@ static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumn ids.num = 1; ids.ids[0] = *pIndex; - if (pIndex->columnIndex >= pMeterMeta->numOfColumns || pIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (pIndex->columnIndex >= tscGetNumOfColumns(pTableMeta) || pIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { ids.num = 0; } @@ -1362,16 +1374,18 @@ void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex } static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColumnIndex* pIndex, int32_t startPos) { - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, pIndex->tableIndex); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); int32_t numOfTotalColumns = 0; - STableMeta* pMeterMeta = pMeterMetaInfo->pMeterMeta; - SSchema* pSchema = tsGetSchema(pMeterMeta); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + SSchema* pSchema = tscGetTableSchema(pTableMeta); - if (UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) { - numOfTotalColumns = pMeterMeta->numOfColumns + pMeterMeta->numOfTags; + STableComInfo tinfo = tscGetTableInfo(pTableMeta); + + if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + numOfTotalColumns = tinfo.numOfColumns + tinfo.numOfTags; } else { - numOfTotalColumns = pMeterMeta->numOfColumns; + numOfTotalColumns = tinfo.numOfColumns; } for (int32_t j = 0; j < numOfTotalColumns; ++j) { @@ -1383,7 +1397,7 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum ids.ids[0] = *pIndex; // tag columns do not add to source list - ids.num = (j >= pMeterMeta->numOfColumns) ? 0 : 1; + ids.num = (j >= tscGetNumOfColumns(pTableMeta)) ? 0 : 1; insertResultField(pQueryInfo, startPos + j, &ids, pSchema[j].bytes, pSchema[j].type, pSchema[j].name, pExpr); } @@ -1427,10 +1441,10 @@ int32_t addProjectionExprAndResultField(SQueryInfo* pQueryInfo, tSQLExprItem* pI pQueryInfo->type = TSDB_QUERY_TYPE_STABLE_QUERY; tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, true); } else { - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, index.tableIndex); - STableMeta* pMeterMeta = pMeterMetaInfo->pMeterMeta; + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - if (index.columnIndex >= pMeterMeta->numOfColumns && UTIL_METER_IS_NOMRAL_METER(pMeterMetaInfo)) { + if (index.columnIndex >= tscGetNumOfColumns(pTableMeta) && UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } @@ -1488,7 +1502,7 @@ static int32_t setExprInfoForFunctions(SQueryInfo* pQueryInfo, SSchema* pSchema, } int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprItem* pItem, bool finalResult) { - SMeterMetaInfo* pMeterMetaInfo = NULL; + STableMetaInfo* pTableMetaInfo = NULL; int32_t optr = pItem->pNode->nSQLOptr; const char* msg1 = "not support column types"; @@ -1538,10 +1552,10 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt return invalidSqlErrMsg(pQueryInfo->msg, msg3); } - pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, index.tableIndex); + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); // count tag is equalled to count(tbname) - if (index.columnIndex >= pMeterMetaInfo->pMeterMeta->numOfColumns) { + if (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { index.columnIndex = TSDB_TBNAME_COLUMN_INDEX; } @@ -1603,8 +1617,8 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt } // 2. check if sql function can be applied on this column data type - pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, index.tableIndex); - SSchema* pSchema = tsGetColumnSchema(pMeterMetaInfo->pMeterMeta, index.columnIndex); + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); int16_t colType = pSchema->type; if (colType <= TSDB_DATA_TYPE_BOOL || colType >= TSDB_DATA_TYPE_BINARY) { @@ -1637,7 +1651,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt } // functions can not be applied to tags - if (index.columnIndex >= pMeterMetaInfo->pMeterMeta->numOfColumns) { + if (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { return invalidSqlErrMsg(pQueryInfo->msg, msg6); } @@ -1711,10 +1725,10 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt return invalidSqlErrMsg(pQueryInfo->msg, msg4); } - pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, index.tableIndex); - SSchema* pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - for (int32_t j = 0; j < pMeterMetaInfo->pMeterMeta->numOfColumns; ++j) { + for (int32_t j = 0; j < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++j) { index.columnIndex = j; if (setExprInfoForFunctions(pQueryInfo, pSchema, functionID, pItem->aliasName, colIdx++, &index) != 0) { return TSDB_CODE_INVALID_SQL; @@ -1726,11 +1740,11 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt return invalidSqlErrMsg(pQueryInfo->msg, msg3); } - pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, index.tableIndex); - SSchema* pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); // functions can not be applied to tags - if ((index.columnIndex >= pMeterMetaInfo->pMeterMeta->numOfColumns) || (index.columnIndex < 0)) { + if ((index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) || (index.columnIndex < 0)) { return invalidSqlErrMsg(pQueryInfo->msg, msg6); } @@ -1745,10 +1759,10 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt int32_t numOfFields = 0; for (int32_t j = 0; j < pQueryInfo->numOfTables; ++j) { - pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, j); - SSchema* pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, j); + SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - for (int32_t i = 0; i < pMeterMetaInfo->pMeterMeta->numOfColumns; ++i) { + for (int32_t i = 0; i < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++i) { SColumnIndex index = {.tableIndex = j, .columnIndex = i}; if (setExprInfoForFunctions(pQueryInfo, pSchema, functionID, pItem->aliasName, colIdx + i + j, &index) != 0) { @@ -1756,7 +1770,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt } } - numOfFields += pMeterMetaInfo->pMeterMeta->numOfColumns; + numOfFields += tscGetNumOfColumns(pTableMetaInfo->pTableMeta); } return TSDB_CODE_SUCCESS; @@ -1782,11 +1796,11 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt return invalidSqlErrMsg(pQueryInfo->msg, msg3); } - pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, index.tableIndex); - SSchema* pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); // functions can not be applied to tags - if (index.columnIndex >= pMeterMetaInfo->pMeterMeta->numOfColumns) { + if (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { return invalidSqlErrMsg(pQueryInfo->msg, msg6); } @@ -1916,10 +1930,10 @@ static bool isTablenameToken(SSQLToken* token) { } static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SSQLToken* pToken) { - STableMeta* pMeterMeta = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, index)->pMeterMeta; + STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, index)->pTableMeta; - int32_t numOfCols = pMeterMeta->numOfColumns + pMeterMeta->numOfTags; - SSchema* pSchema = tsGetSchema(pMeterMeta); + int32_t numOfCols = tscGetNumOfColumns(pTableMeta) + tscGetNumOfTags(pTableMeta); + SSchema* pSchema = tscGetTableSchema(pTableMeta); int16_t columnIndex = COLUMN_INDEX_INITIAL_VAL; @@ -1991,8 +2005,8 @@ int32_t getMeterIndex(SSQLToken* pTableToken, SQueryInfo* pQueryInfo, SColumnInd char tableName[TSDB_TABLE_ID_LEN + 1] = {0}; for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, i); - extractTableName(pMeterMetaInfo->name, tableName); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); + extractTableName(pTableMetaInfo->name, tableName); if (strncasecmp(tableName, pTableToken->z, pTableToken->n) == 0 && strlen(tableName) == pTableToken->n) { pIndex->tableIndex = i; @@ -2019,7 +2033,7 @@ int32_t getTableIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIn } int32_t getColumnIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { - if (pQueryInfo->pMeterInfo == NULL || pQueryInfo->numOfTables == 0) { + if (pQueryInfo->pTableMetaInfo == NULL || pQueryInfo->numOfTables == 0) { return TSDB_CODE_INVALID_SQL; } @@ -2115,7 +2129,7 @@ int32_t changeFunctionID(int32_t optr, int16_t* functionId) { int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { SSqlCmd* pCmd = &pSql->cmd; - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); assert(pCmd->numOfClause == 1); pCmd->command = TSDB_SQL_SHOW; @@ -2151,7 +2165,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - int32_t ret = setObjFullName(pMeterMetaInfo->name, getAccountId(pSql), pDbPrefixToken, NULL, NULL); + int32_t ret = setObjFullName(pTableMetaInfo->name, getAccountId(pSql), pDbPrefixToken, NULL, NULL); if (ret != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -2235,13 +2249,13 @@ bool validateIpAddress(const char* ip, size_t size) { } int32_t tscTansformSQLFunctionForSTableQuery(SQueryInfo* pQueryInfo) { - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (pMeterMetaInfo->pMeterMeta == NULL || !UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) { + if (pTableMetaInfo->pTableMeta == NULL || !UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { return TSDB_CODE_INVALID_SQL; } - assert(pMeterMetaInfo->pMeterMeta->numOfTags >= 0); + assert(tscGetNumOfTags(pTableMetaInfo->pTableMeta) >= 0); int16_t bytes = 0; int16_t type = 0; @@ -2252,7 +2266,7 @@ int32_t tscTansformSQLFunctionForSTableQuery(SQueryInfo* pQueryInfo) { int16_t functionId = aAggs[pExpr->functionId].stableFuncId; int32_t colIndex = pExpr->colInfo.colIdx; - SSchema* pSrcSchema = tsGetColumnSchema(pMeterMetaInfo->pMeterMeta, colIndex); + SSchema* pSrcSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, colIndex); if ((functionId >= TSDB_FUNC_SUM && functionId <= TSDB_FUNC_TWA) || (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) || @@ -2274,14 +2288,14 @@ int32_t tscTansformSQLFunctionForSTableQuery(SQueryInfo* pQueryInfo) { /* transfer the field-info back to original input format */ void tscRestoreSQLFunctionForMetricQuery(SQueryInfo* pQueryInfo) { - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - if (!UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) { + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { return; } for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); - SSchema* pSchema = tsGetColumnSchema(pMeterMetaInfo->pMeterMeta, pExpr->colInfo.colIdx); + SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIdx); // if (/*(pExpr->functionId >= TSDB_FUNC_FIRST_DST && pExpr->functionId <= TSDB_FUNC_LAST_DST) || // (pExpr->functionId >= TSDB_FUNC_SUM && pExpr->functionId <= TSDB_FUNC_MAX) || @@ -2371,7 +2385,7 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) { } void updateTagColumnIndex(SQueryInfo* pQueryInfo, int32_t tableIndex) { - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, tableIndex); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); /* * update tags column index for group by tags @@ -2381,8 +2395,8 @@ void updateTagColumnIndex(SQueryInfo* pQueryInfo, int32_t tableIndex) { for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) { int32_t index = pQueryInfo->groupbyExpr.columnInfo[i].colIdx; - for (int32_t j = 0; j < pMeterMetaInfo->numOfTags; ++j) { - int32_t tagColIndex = pMeterMetaInfo->tagColumnIndex[j]; + for (int32_t j = 0; j < pTableMetaInfo->numOfTags; ++j) { + int32_t tagColIndex = pTableMetaInfo->tagColumnIndex[j]; if (tagColIndex == index) { pQueryInfo->groupbyExpr.columnInfo[i].colIdx = j; break; @@ -2400,12 +2414,12 @@ void updateTagColumnIndex(SQueryInfo* pQueryInfo, int32_t tableIndex) { } // not belongs to this table - if (pExpr->uid != pMeterMetaInfo->pMeterMeta->uid) { + if (pExpr->uid != pTableMetaInfo->pTableMeta->uid) { continue; } - for (int32_t j = 0; j < pMeterMetaInfo->numOfTags; ++j) { - if (pExpr->colInfo.colIdx == pMeterMetaInfo->tagColumnIndex[j]) { + for (int32_t j = 0; j < pTableMetaInfo->numOfTags; ++j) { + if (pExpr->colInfo.colIdx == pTableMetaInfo->tagColumnIndex[j]) { pExpr->colInfo.colIdx = j; break; } @@ -2421,17 +2435,17 @@ void updateTagColumnIndex(SQueryInfo* pQueryInfo, int32_t tableIndex) { assert(pJoinInfo->left.uid != pJoinInfo->right.uid); // the join condition expression node belongs to this table(super table) - if (pMeterMetaInfo->pMeterMeta->uid == pJoinInfo->left.uid) { - for (int32_t i = 0; i < pMeterMetaInfo->numOfTags; ++i) { - if (pJoinInfo->left.tagCol == pMeterMetaInfo->tagColumnIndex[i]) { + if (pTableMetaInfo->pTableMeta->uid == pJoinInfo->left.uid) { + for (int32_t i = 0; i < pTableMetaInfo->numOfTags; ++i) { + if (pJoinInfo->left.tagCol == pTableMetaInfo->tagColumnIndex[i]) { pJoinInfo->left.tagCol = i; } } } - if (pMeterMetaInfo->pMeterMeta->uid == pJoinInfo->right.uid) { - for (int32_t i = 0; i < pMeterMetaInfo->numOfTags; ++i) { - if (pJoinInfo->right.tagCol == pMeterMetaInfo->tagColumnIndex[i]) { + if (pTableMetaInfo->pTableMeta->uid == pJoinInfo->right.uid) { + for (int32_t i = 0; i < pTableMetaInfo->numOfTags; ++i) { + if (pJoinInfo->right.tagCol == pTableMetaInfo->tagColumnIndex[i]) { pJoinInfo->right.tagCol = i; } } @@ -2447,7 +2461,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* const char* msg9 = "tags not allowed for table query"; // todo : handle two meter situation - SMeterMetaInfo* pMeterMetaInfo = NULL; + STableMetaInfo* pTableMetaInfo = NULL; if (pList == NULL) { return TSDB_CODE_SUCCESS; @@ -2458,9 +2472,9 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* return invalidSqlErrMsg(pQueryInfo->msg, msg1); } - STableMeta* pMeterMeta = NULL; + STableMeta* pTableMeta = NULL; SSchema* pSchema = NULL; - SSchema s = tsGetTbnameColumnSchema(); + SSchema s = tscGetTbnameColumnSchema(); int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL; @@ -2480,28 +2494,28 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* tableIndex = index.tableIndex; - pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, index.tableIndex); - pMeterMeta = pMeterMetaInfo->pMeterMeta; + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + pTableMeta = pTableMetaInfo->pTableMeta; if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { pSchema = &s; } else { - pSchema = tsGetColumnSchema(pMeterMeta, index.columnIndex); + pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); } bool groupTag = false; - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX || index.columnIndex >= pMeterMeta->numOfColumns) { + if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX || index.columnIndex >= tscGetNumOfColumns(pTableMeta)) { groupTag = true; } if (groupTag) { - if (!UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) { + if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { return invalidSqlErrMsg(pQueryInfo->msg, msg9); } int32_t relIndex = index.columnIndex; if (index.columnIndex != TSDB_TBNAME_COLUMN_INDEX) { - relIndex -= pMeterMeta->numOfColumns; + relIndex -= tscGetNumOfColumns(pTableMeta); } pQueryInfo->groupbyExpr.columnInfo[i] = @@ -2561,9 +2575,9 @@ static int32_t doExtractColumnFilterInfo(SQueryInfo* pQueryInfo, SColumnFilterIn const char* msg = "not supported filter condition"; tSQLExpr* pRight = pExpr->pRight; - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, columnIndex->tableIndex); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, columnIndex->tableIndex); - SSchema* pSchema = tsGetColumnSchema(pMeterMetaInfo->pMeterMeta, columnIndex->columnIndex); + SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, columnIndex->columnIndex); int16_t colType = pSchema->type; if (colType >= TSDB_DATA_TYPE_TINYINT && colType <= TSDB_DATA_TYPE_BIGINT) { @@ -2806,10 +2820,10 @@ enum { }; static int32_t extractColumnFilterInfo(SQueryInfo* pQueryInfo, SColumnIndex* pIndex, tSQLExpr* pExpr, int32_t sqlOptr) { - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, pIndex->tableIndex); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); - STableMeta* pMeterMeta = pMeterMetaInfo->pMeterMeta; - SSchema* pSchema = tsGetColumnSchema(pMeterMeta, pIndex->columnIndex); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex); const char* msg1 = "non binary column not support like operator"; const char* msg2 = "binary column not support this operator"; @@ -2966,24 +2980,24 @@ static int32_t getJoinCondInfo(SQueryInfo* pQueryInfo, tSQLExpr* pExpr) { return TSDB_CODE_INVALID_SQL; } - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, index.tableIndex); - int16_t tagColIndex = index.columnIndex - pMeterMetaInfo->pMeterMeta->numOfColumns; + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + int16_t tagColIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - pLeft->uid = pMeterMetaInfo->pMeterMeta->uid; + pLeft->uid = pTableMetaInfo->pTableMeta->uid; pLeft->tagCol = tagColIndex; - strcpy(pLeft->tableId, pMeterMetaInfo->name); + strcpy(pLeft->tableId, pTableMetaInfo->name); index = (SColumnIndex)COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(&pExpr->pRight->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } - pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, index.tableIndex); - tagColIndex = index.columnIndex - pMeterMetaInfo->pMeterMeta->numOfColumns; + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + tagColIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - pRight->uid = pMeterMetaInfo->pMeterMeta->uid; + pRight->uid = pTableMetaInfo->pTableMeta->uid; pRight->tagCol = tagColIndex; - strcpy(pRight->tableId, pMeterMetaInfo->name); + strcpy(pRight->tableId, pTableMetaInfo->name); pTagCond->joinInfo.hasJoin = true; return TSDB_CODE_SUCCESS; @@ -3035,8 +3049,9 @@ static int32_t validateSQLExpr(tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnL } // if column is timestamp, bool, binary, nchar, not support arithmetic, so return invalid sql - STableMeta* pMeterMeta = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, index.tableIndex)->pMeterMeta; - SSchema* pSchema = tsGetSchema(pMeterMeta) + index.columnIndex; + STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, index.tableIndex)->pTableMeta; + SSchema* pSchema = tscGetTableSchema(pTableMeta) + index.columnIndex; + if ((pSchema->type == TSDB_DATA_TYPE_TIMESTAMP) || (pSchema->type == TSDB_DATA_TYPE_BOOL) || (pSchema->type == TSDB_DATA_TYPE_BINARY) || (pSchema->type == TSDB_DATA_TYPE_NCHAR)) { return TSDB_CODE_INVALID_SQL; @@ -3183,12 +3198,12 @@ static bool validateJoinExprNode(SQueryInfo* pQueryInfo, tSQLExpr* pExpr, SColum } // todo extract function - SMeterMetaInfo* pLeftMeterMeta = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, pLeftIndex->tableIndex); - SSchema* pLeftSchema = tsGetSchema(pLeftMeterMeta->pMeterMeta); + STableMetaInfo* pLeftMeterMeta = tscGetMetaInfo(pQueryInfo, pLeftIndex->tableIndex); + SSchema* pLeftSchema = tscGetTableSchema(pLeftMeterMeta->pTableMeta); int16_t leftType = pLeftSchema[pLeftIndex->columnIndex].type; - SMeterMetaInfo* pRightMeterMeta = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, rightIndex.tableIndex); - SSchema* pRightSchema = tsGetSchema(pRightMeterMeta->pMeterMeta); + STableMetaInfo* pRightMeterMeta = tscGetMetaInfo(pQueryInfo, rightIndex.tableIndex); + SSchema* pRightSchema = tscGetTableSchema(pRightMeterMeta->pTableMeta); int16_t rightType = pRightSchema[rightIndex.columnIndex].type; if (leftType != rightType) { @@ -3203,7 +3218,7 @@ static bool validateJoinExprNode(SQueryInfo* pQueryInfo, tSQLExpr* pExpr, SColum } // table to table/ super table to super table are allowed - if (UTIL_METER_IS_SUPERTABLE(pLeftMeterMeta) != UTIL_METER_IS_SUPERTABLE(pRightMeterMeta)) { + if (UTIL_TABLE_IS_SUPERTABLE(pLeftMeterMeta) != UTIL_TABLE_IS_SUPERTABLE(pRightMeterMeta)) { invalidSqlErrMsg(pQueryInfo->msg, msg5); return false; } @@ -3259,8 +3274,8 @@ static int32_t handleExprInQueryCond(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, S assert(isExprDirectParentOfLeaftNode(*pExpr)); - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, index.tableIndex); - STableMeta* pMeterMeta = pMeterMetaInfo->pMeterMeta; + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; if (index.columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { // query on time range if (!validateJoinExprNode(pQueryInfo, *pExpr, &index)) { @@ -3283,10 +3298,10 @@ static int32_t handleExprInQueryCond(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, S *pExpr = NULL; // remove this expression *type = TSQL_EXPR_TS; - } else if (index.columnIndex >= pMeterMeta->numOfColumns || + } else if (index.columnIndex >= tscGetNumOfColumns(pTableMeta) || index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { // query on tags // check for tag query condition - if (UTIL_METER_IS_NOMRAL_METER(pMeterMetaInfo)) { + if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } @@ -3296,7 +3311,7 @@ static int32_t handleExprInQueryCond(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, S return TSDB_CODE_INVALID_SQL; } - SSchema* pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); + SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); if ((!isTablenameToken(&pLeft->colInfo)) && pSchema[index.columnIndex].type != TSDB_DATA_TYPE_BINARY && pSchema[index.columnIndex].type != TSDB_DATA_TYPE_NCHAR) { @@ -3494,10 +3509,10 @@ static int32_t setTableCondForMetricQuery(SQueryInfo* pQueryInfo, const char* ac return TSDB_CODE_SUCCESS; } - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, tableCondIndex); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableCondIndex); STagCond* pTagCond = &pQueryInfo->tagCond; - pTagCond->tbnameCond.uid = pMeterMetaInfo->pMeterMeta->uid; + pTagCond->tbnameCond.uid = pTableMetaInfo->pTableMeta->uid; assert(pExpr->nSQLOptr == TK_LIKE || pExpr->nSQLOptr == TK_IN); @@ -3527,8 +3542,9 @@ static int32_t setTableCondForMetricQuery(SQueryInfo* pQueryInfo, const char* ac } num = j; - SSQLToken dbToken = extractDBName(pMeterMetaInfo->name, db); - + char* name = extractDBName(pTableMetaInfo->name, db); + SSQLToken dbToken = {.type = TK_STRING, .z = name, .n = strlen(name)}; + for (int32_t i = 0; i < num; ++i) { if (i >= 1) { taosStringBuilderAppendStringLen(&sb1, TBNAME_LIST_SEP, 1); @@ -3604,15 +3620,15 @@ static int32_t getTimeRangeFromExpr(SQueryInfo* pQueryInfo, tSQLExpr* pExpr) { return TSDB_CODE_INVALID_SQL; } - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, index.tableIndex); - STableMeta* pMeterMeta = pMeterMetaInfo->pMeterMeta; - + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); + tSQLExpr* pRight = pExpr->pRight; TSKEY stime = 0; TSKEY etime = INT64_MAX; - if (getTimeRange(&stime, &etime, pRight, pExpr->nSQLOptr, pMeterMeta->precision) != TSDB_CODE_SUCCESS) { + if (getTimeRange(&stime, &etime, pRight, pExpr->nSQLOptr, tinfo.precision) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(pQueryInfo->msg, msg0); } @@ -3642,8 +3658,8 @@ static int32_t validateJoinExpr(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr) { } } - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - if (UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) { // for stable join, tag columns + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { // for stable join, tag columns // must be present for join if (pCondExpr->pJoinExpr == NULL) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); @@ -3680,20 +3696,20 @@ static void cleanQueryExpr(SCondExpr* pCondExpr) { } static void doAddJoinTagsColumnsIntoTagList(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr) { - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - if (QUERY_IS_JOIN_QUERY(pQueryInfo->type) && UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) { + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + if (QUERY_IS_JOIN_QUERY(pQueryInfo->type) && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { SColumnIndex index = {0}; getColumnIndexByName(&pCondExpr->pJoinExpr->pLeft->colInfo, pQueryInfo, &index); - pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, index.tableIndex); + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - int32_t columnInfo = index.columnIndex - pMeterMetaInfo->pMeterMeta->numOfColumns; + int32_t columnInfo = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); addRequiredTagColumn(pQueryInfo, columnInfo, index.tableIndex); getColumnIndexByName(&pCondExpr->pJoinExpr->pRight->colInfo, pQueryInfo, &index); - pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, index.tableIndex); + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - columnInfo = index.columnIndex - pMeterMetaInfo->pMeterMeta->numOfColumns; + columnInfo = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); addRequiredTagColumn(pQueryInfo, columnInfo, index.tableIndex); } } @@ -3705,7 +3721,7 @@ static int32_t getTagQueryCondExpr(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr, for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { tSQLExpr* p1 = extractExprForSTable(pExpr, pQueryInfo, i); - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, i); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); char c[TSDB_MAX_TAGS_LEN] = {0}; char* str = c; @@ -3714,7 +3730,7 @@ static int32_t getTagQueryCondExpr(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr, return ret; } - tsSetMetricQueryCond(&pQueryInfo->tagCond, pMeterMetaInfo->pMeterMeta->uid, c); + tsSetMetricQueryCond(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid, c); doCompactQueryExpr(pExpr); tSQLExprDestroy(p1); @@ -4019,7 +4035,7 @@ int32_t parseFillClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL) { static void setDefaultOrderInfo(SQueryInfo* pQueryInfo) { /* set default timestamp order information for all queries */ pQueryInfo->order.order = TSQL_SO_ASC; - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); if (isTopBottomQuery(pQueryInfo)) { pQueryInfo->order.order = TSQL_SO_ASC; @@ -4029,7 +4045,7 @@ static void setDefaultOrderInfo(SQueryInfo* pQueryInfo) { } /* for metric query, set default ascending order for group output */ - if (UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) { + if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { pQueryInfo->groupbyExpr.orderType = TSQL_SO_ASC; } } @@ -4041,7 +4057,7 @@ int32_t parseOrderbyClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema const char* msg3 = "only support order by primary timestamp and first tag in groupby clause"; setDefaultOrderInfo(pQueryInfo); - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); if (pQuerySql->pSortOrder == NULL) { return TSDB_CODE_SUCCESS; @@ -4055,7 +4071,7 @@ int32_t parseOrderbyClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema * * for super table query, the order option must be less than 3. */ - if (UTIL_METER_IS_NOMRAL_METER(pMeterMetaInfo)) { + if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { if (pSortorder->nExpr > 1) { return invalidSqlErrMsg(pQueryInfo->msg, msg0); } @@ -4076,7 +4092,7 @@ int32_t parseOrderbyClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema SSQLToken columnName = {pVar->nLen, pVar->nType, pVar->pz}; SColumnIndex index = {0}; - if (UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) { // metric query + if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { // metric query if (getColumnIndexByName(&columnName, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } @@ -4084,8 +4100,8 @@ int32_t parseOrderbyClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema bool orderByTags = false; bool orderByTS = false; - if (index.columnIndex >= pMeterMetaInfo->pMeterMeta->numOfColumns) { - int32_t relTagIndex = index.columnIndex - pMeterMetaInfo->pMeterMeta->numOfColumns; + if (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { + int32_t relTagIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); if (relTagIndex == pQueryInfo->groupbyExpr.columnInfo[0].colIdx) { orderByTags = true; } @@ -4105,7 +4121,7 @@ int32_t parseOrderbyClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema if (pSortorder->nExpr == 1) { if (orderByTags) { - pQueryInfo->groupbyExpr.orderIndex = index.columnIndex - pMeterMetaInfo->pMeterMeta->numOfColumns; + pQueryInfo->groupbyExpr.orderIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); pQueryInfo->groupbyExpr.orderType = pQuerySql->pSortOrder->a[0].sortOrder; } else if (isTopBottomQuery(pQueryInfo)) { /* order of top/bottom query in interval is not valid */ @@ -4128,7 +4144,7 @@ int32_t parseOrderbyClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema if (pSortorder->nExpr == 2) { if (orderByTags) { - pQueryInfo->groupbyExpr.orderIndex = index.columnIndex - pMeterMetaInfo->pMeterMeta->numOfColumns; + pQueryInfo->groupbyExpr.orderIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); pQueryInfo->groupbyExpr.orderType = pQuerySql->pSortOrder->a[0].sortOrder; } else { pQueryInfo->order.order = pSortorder->a[0].sortOrder; @@ -4193,32 +4209,32 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { SAlterTableSQL* pAlterSQL = pInfo->pAlterInfo; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, DEFAULT_TABLE_INDEX); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, DEFAULT_TABLE_INDEX); if (tscValidateName(&(pAlterSQL->name)) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } - if (setMeterID(pMeterMetaInfo, &(pAlterSQL->name), pSql) != TSDB_CODE_SUCCESS) { + if (setMeterID(pTableMetaInfo, &(pAlterSQL->name), pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); } - int32_t ret = tscGetMeterMeta(pSql, pMeterMetaInfo); + int32_t ret = tscGetTableMeta(pSql, pTableMetaInfo); if (ret != TSDB_CODE_SUCCESS) { return ret; } - STableMeta* pMeterMeta = pMeterMetaInfo->pMeterMeta; + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; if (pAlterSQL->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) { - if (UTIL_METER_IS_NOMRAL_METER(pMeterMetaInfo)) { + if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { return invalidSqlErrMsg(pQueryInfo->msg, msg3); } - } else if ((pAlterSQL->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) && (UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo))) { + } else if ((pAlterSQL->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) && (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo))) { return invalidSqlErrMsg(pQueryInfo->msg, msg4); } else if ((pAlterSQL->type == TSDB_ALTER_TABLE_ADD_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_DROP_COLUMN) && - UTIL_METER_IS_CREATE_FROM_METRIC(pMeterMetaInfo)) { + UTIL_TABLE_CREATE_FROM_STABLE(pTableMetaInfo)) { return invalidSqlErrMsg(pQueryInfo->msg, msg6); } @@ -4240,7 +4256,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg4 = "illegal tag name"; const char* msg5 = "primary tag cannot be dropped"; - if (pMeterMeta->numOfTags == 1) { + if (tscGetNumOfTags(pTableMeta) == 1) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } @@ -4261,7 +4277,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { return TSDB_CODE_INVALID_SQL; } - if (index.columnIndex < pMeterMeta->numOfColumns) { + if (index.columnIndex < tscGetNumOfColumns(pTableMeta)) { return invalidSqlErrMsg(pQueryInfo->msg, msg4); } else if (index.columnIndex == 0) { return invalidSqlErrMsg(pQueryInfo->msg, msg5); @@ -4327,11 +4343,11 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { return TSDB_CODE_INVALID_SQL; } - if (columnIndex.columnIndex < pMeterMeta->numOfColumns) { + if (columnIndex.columnIndex < tscGetNumOfColumns(pTableMeta)) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); } - SSchema* pTagsSchema = tsGetColumnSchema(pMeterMetaInfo->pMeterMeta, columnIndex.columnIndex); + SSchema* pTagsSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, columnIndex.columnIndex); if (tVariantDump(&pVarList->a[1].pVar, pAlterSQL->tagData.data /*pCmd->payload*/, pTagsSchema->type) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); @@ -4366,7 +4382,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg4 = "illegal column name"; const char* msg3 = "primary timestamp column cannot be dropped"; - if (pMeterMeta->numOfColumns == TSDB_MIN_COLUMNS) { // + if (tscGetNumOfColumns(pTableMeta) == TSDB_MIN_COLUMNS) { // return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -4586,7 +4602,7 @@ bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo) { } int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL* pQuerySql, SSqlObj* pSql) { - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); const char* msg0 = "soffset/offset can not be less than 0"; const char* msg1 = "slimit/soffset only available for STable query"; @@ -4611,7 +4627,7 @@ int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL* return TSDB_CODE_SUCCESS; } - if (UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) { + if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { bool queryOnTags = false; if (tscQueryOnlyMetricTags(pQueryInfo, &queryOnTags) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; @@ -4650,8 +4666,8 @@ int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL* } // No tables included. No results generated. Query results are empty. - SSuperTableMeta* pMetricMeta = pMeterMetaInfo->pMetricMeta; - if (pMeterMetaInfo->pMeterMeta == NULL || pMetricMeta == NULL || pMetricMeta->numOfTables == 0) { + SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta; + if (pTableMetaInfo->pTableMeta == NULL || pMetricMeta == NULL || pMetricMeta->numOfTables == 0) { tscTrace("%p no table in metricmeta, no output result", pSql); pQueryInfo->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; } @@ -4807,10 +4823,10 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClau SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, num - 1); if (pExpr->functionId != TSDB_FUNC_TAG) { - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, tableIndex); - int16_t columnInfo = tscGetJoinTagColIndexByUid(&pQueryInfo->tagCond, pMeterMetaInfo->pMeterMeta->uid); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); + int16_t columnInfo = tscGetJoinTagColIndexByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid); SColumnIndex index = {.tableIndex = 0, .columnIndex = columnInfo}; - SSchema* pSchema = tsGetTagSchema(pMeterMetaInfo->pMeterMeta); + SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); int16_t type = pSchema[index.columnIndex].type; int16_t bytes = pSchema[index.columnIndex].bytes; @@ -4844,9 +4860,9 @@ static void doLimitOutputNormalColOfGroupby(SSqlExpr* pExpr) { void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex) { int32_t index = pQueryInfo->groupbyExpr.columnInfo[tagIndex].colIdx; - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - SSchema* pSchema = tsGetColumnSchema(pMeterMetaInfo->pMeterMeta, index); + SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index); SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = index}; SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, pQueryInfo->exprsInfo.numOfExprs, TSDB_FUNC_PRJ, &colIndex, @@ -4878,8 +4894,8 @@ static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) { } } - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - SSchema* pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); @@ -5054,9 +5070,9 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo) { static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) { const char* msg2 = "interval not allowed in group by normal column"; - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - SSchema* pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); + SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); int16_t bytes = 0; int16_t type = 0; char* name = NULL; @@ -5070,7 +5086,7 @@ static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) { bytes = TSDB_TABLE_NAME_LEN; name = TSQL_TBNAME_L; } else { - colIndex = (TSDB_COL_IS_TAG(pColIndex->flag)) ? pMeterMetaInfo->pMeterMeta->numOfColumns + pColIndex->colIdx + colIndex = (TSDB_COL_IS_TAG(pColIndex->flag)) ? tscGetNumOfColumns(pTableMetaInfo->pTableMeta) + pColIndex->colIdx : pColIndex->colIdx; type = pSchema[colIndex].type; @@ -5364,7 +5380,7 @@ int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* p SSqlCmd* pCmd = &pSql->cmd; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, subClauseIndex); - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); SCreateTableSQL* pCreateTable = pInfo->pCreateTableInfo; @@ -5380,7 +5396,7 @@ int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* p return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - if (setMeterID(pMeterMetaInfo, pzTableName, pSql) != TSDB_CODE_SUCCESS) { + if (setMeterID(pTableMetaInfo, pzTableName, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -5419,13 +5435,13 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); // two table: the first one is for current table, and the secondary is for the super table. - tscAddEmptyMeterMetaInfo(pQueryInfo); + tscAddEmptyMetaInfo(pQueryInfo); assert(pQueryInfo->numOfTables == 2); const int32_t TABLE_INDEX = 0; const int32_t STABLE_INDEX = 1; - SMeterMetaInfo* pStableMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, STABLE_INDEX); + STableMetaInfo* pStableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX); // super table name, create table by using dst SSQLToken* pToken = &(pCreateTable->usingInfo.stableName); @@ -5442,17 +5458,17 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { strncpy(pCreateTable->usingInfo.tagdata.name, pStableMeterMetaInfo->name, TSDB_TABLE_ID_LEN); tVariantList* pList = pInfo->pCreateTableInfo->usingInfo.pTagVals; - int32_t code = tscGetMeterMeta(pSql, pStableMeterMetaInfo); + int32_t code = tscGetTableMeta(pSql, pStableMeterMetaInfo); if (code != TSDB_CODE_SUCCESS) { return code; } - if (pStableMeterMetaInfo->pMeterMeta->numOfTags != pList->nExpr) { + if (tscGetNumOfTags(pStableMeterMetaInfo->pTableMeta) != pList->nExpr) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); } // too long tag values will return invalid sql, not be truncated automatically - SSchema* pTagSchema = tsGetTagSchema(pStableMeterMetaInfo->pMeterMeta); + SSchema* pTagSchema = tscGetTableTagSchema(pStableMeterMetaInfo->pTableMeta); char* tagVal = pCreateTable->usingInfo.tagdata.data; for (int32_t i = 0; i < pList->nExpr; ++i) { @@ -5475,7 +5491,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - SMeterMetaInfo* pTableMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, TABLE_INDEX); + STableMetaInfo* pTableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, TABLE_INDEX); int32_t ret = setMeterID(pTableMeterMetaInfo, &pInfo->pCreateTableInfo->name, pSql); if (ret != TSDB_CODE_SUCCESS) { return ret; @@ -5496,7 +5512,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { assert(pQueryInfo->numOfTables == 1); SCreateTableSQL* pCreateTable = pInfo->pCreateTableInfo; - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); // if sql specifies db, use it, otherwise use default db SSQLToken* pzTableName = &(pCreateTable->name); @@ -5514,16 +5530,16 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } - if (setMeterID(pMeterMetaInfo, &srcToken, pSql) != TSDB_CODE_SUCCESS) { + if (setMeterID(pTableMetaInfo, &srcToken, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); } - int32_t code = tscGetMeterMeta(pSql, pMeterMetaInfo); + int32_t code = tscGetTableMeta(pSql, pTableMetaInfo); if (code != TSDB_CODE_SUCCESS) { return code; } - bool isSTable = UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo); + bool isSTable = UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo); if (parseSelectClause(&pSql->cmd, 0, pQuerySql->pSelection, isSTable) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } @@ -5545,7 +5561,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { } // set the created table[stream] name - if (setMeterID(pMeterMetaInfo, pzTableName, pSql) != TSDB_CODE_SUCCESS) { + if (setMeterID(pTableMetaInfo, pzTableName, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } @@ -5603,9 +5619,9 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { SSqlCmd* pCmd = &pSql->cmd; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, index); - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - if (pMeterMetaInfo == NULL) { - pMeterMetaInfo = tscAddEmptyMeterMetaInfo(pQueryInfo); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + if (pTableMetaInfo == NULL) { + pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); } // too many result columns not support order by in query @@ -5648,17 +5664,17 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { } if (pQueryInfo->numOfTables <= i) { // more than one table - tscAddEmptyMeterMetaInfo(pQueryInfo); + tscAddEmptyMetaInfo(pQueryInfo); } - SMeterMetaInfo* pMeterInfo1 = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, i); + STableMetaInfo* pMeterInfo1 = tscGetMetaInfo(pQueryInfo, i); SSQLToken t = {.type = TSDB_DATA_TYPE_BINARY, .n = pTableItem->nLen, .z = pTableItem->pz}; if (setMeterID(pMeterInfo1, &t, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - code = tscGetMeterMeta(pSql, pMeterInfo1); + code = tscGetTableMeta(pSql, pMeterInfo1); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -5671,7 +5687,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return TSDB_CODE_INVALID_SQL; } - bool isSTable = UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo); + bool isSTable = UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo); if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } @@ -5687,19 +5703,20 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { } // set order by info - if (parseOrderbyClause(pQueryInfo, pQuerySql, tsGetSchema(pMeterMetaInfo->pMeterMeta)) != TSDB_CODE_SUCCESS) { + if (parseOrderbyClause(pQueryInfo, pQuerySql, tscGetTableSchema(pTableMetaInfo->pTableMeta)) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } // set where info + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); + if (pQuerySql->pWhere != NULL) { if (parseWhereClause(pQueryInfo, &pQuerySql->pWhere, pSql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } pQuerySql->pWhere = NULL; - - if (pMeterMetaInfo->pMeterMeta->precision == TSDB_TIME_PRECISION_MILLI) { + if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { pQueryInfo->stime = pQueryInfo->stime / 1000; pQueryInfo->etime = pQueryInfo->etime / 1000; } @@ -5710,8 +5727,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { // user does not specified the query time window, twa is not allowed in such case. if ((pQueryInfo->stime == 0 || pQueryInfo->etime == INT64_MAX || - (pQueryInfo->etime == INT64_MAX / 1000 && pMeterMetaInfo->pMeterMeta->precision == TSDB_TIME_PRECISION_MILLI)) && - tscIsTWAQuery(pQueryInfo)) { + (pQueryInfo->etime == INT64_MAX / 1000 && tinfo.precision == TSDB_TIME_PRECISION_MILLI)) && tscIsTWAQuery(pQueryInfo)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9); } diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index d412ac18c86988665a2f5a1e1dad688d4bc45700..f4fb761ed9efa1057042e039f617252fda6f9a82 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -19,6 +19,68 @@ #include "ttokendef.h" #include "taosdef.h" #include "tutil.h" +#include "tsclient.h" + +int32_t tscGetNumOfTags(const STableMeta* pTableMeta) { + assert(pTableMeta != NULL); + + STableComInfo tinfo = tscGetTableInfo(pTableMeta); + + if (pTableMeta->tableType == TSDB_NORMAL_TABLE) { + assert(tinfo.numOfTags == 0); + return 0; + } + + if (pTableMeta->tableType == TSDB_SUPER_TABLE || pTableMeta->tableType == TSDB_CHILD_TABLE) { + assert(tinfo.numOfTags > 0); + return tinfo.numOfTags; + } + + assert(tinfo.numOfTags == 0); + return 0; +} + +int32_t tscGetNumOfColumns(const STableMeta* pTableMeta) { + assert(pTableMeta != NULL); + + // table created according to super table, use data from super table + STableComInfo tinfo = tscGetTableInfo(pTableMeta); + return tinfo.numOfColumns; +} + +SSchema *tscGetTableSchema(const STableMeta *pTableMeta) { + assert(pTableMeta != NULL); + + if (pTableMeta->tableType == TSDB_CHILD_TABLE) { + STableMeta* pSTableMeta = pTableMeta->pSTable; + assert (pSTableMeta != NULL); + + return pSTableMeta->schema; + } + + return pTableMeta->schema; +} + +SSchema* tscGetTableTagSchema(const STableMeta* pTableMeta) { + assert(pTableMeta != NULL && (pTableMeta->tableType == TSDB_SUPER_TABLE || pTableMeta->tableType == TSDB_CHILD_TABLE)); + + STableComInfo tinfo = tscGetTableInfo(pTableMeta); + assert(tinfo.numOfTags > 0); + + return tscGetTableColumnSchema(pTableMeta, tinfo.numOfColumns); +} + +STableComInfo tscGetTableInfo(const STableMeta* pTableMeta) { + assert(pTableMeta != NULL); + + if (pTableMeta->tableType == TSDB_CHILD_TABLE) { + assert (pTableMeta->pSTable != NULL); + return pTableMeta->pSTable->tableInfo; + } + + return pTableMeta->tableInfo; +} + bool isValidSchema(struct SSchema* pSchema, int32_t numOfCols) { if (!VALIDNUMOFCOLS(numOfCols)) { @@ -64,34 +126,66 @@ bool isValidSchema(struct SSchema* pSchema, int32_t numOfCols) { return (rowLen <= TSDB_MAX_BYTES_PER_ROW); } -struct SSchema* tsGetSchema(STableMeta* pMeta) { - if (pMeta == NULL) { - return NULL; - } - return tsGetColumnSchema(pMeta, 0); -} - -struct SSchema* tsGetTagSchema(STableMeta* pMeta) { - if (pMeta == NULL || pMeta->numOfTags == 0) { - return NULL; +SSchema* tscGetTableColumnSchema(const STableMeta* pTableMeta, int32_t startCol) { + assert(pTableMeta != NULL); + + SSchema* pSchema = pTableMeta->schema; + + if (pTableMeta->tableType == TSDB_CHILD_TABLE) { + assert (pTableMeta->pSTable != NULL); + pSchema = pTableMeta->pSTable->schema; } - - return tsGetColumnSchema(pMeta, pMeta->numOfColumns); + + return &pSchema[startCol]; } -struct SSchema* tsGetColumnSchema(STableMeta* pMeta, int32_t startCol) { - return (SSchema*)(((char*)pMeta + sizeof(STableMeta)) + startCol * sizeof(SSchema)); +struct SSchema tscGetTbnameColumnSchema() { + struct SSchema s = { + .colId = TSDB_TBNAME_COLUMN_INDEX, + .type = TSDB_DATA_TYPE_BINARY, + .bytes = TSDB_TABLE_NAME_LEN + }; + + strcpy(s.name, TSQL_TBNAME_L); + return s; } -struct SSchema tsGetTbnameColumnSchema() { - struct SSchema s = {.colId = TSDB_TBNAME_COLUMN_INDEX, .type = TSDB_DATA_TYPE_BINARY, .bytes = TSDB_TABLE_NAME_LEN}; - strcpy(s.name, TSQL_TBNAME_L); +STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size) { + assert(pTableMetaMsg != NULL); - return s; + int32_t schemaSize = (pTableMetaMsg->numOfColumns + pTableMetaMsg->numOfTags) * sizeof(SSchema); + STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + schemaSize); + pTableMeta->tableType = pTableMetaMsg->tableType; + + pTableMeta->tableInfo = (STableComInfo) { + .numOfTags = pTableMetaMsg->numOfTags, + .numOfColumns = pTableMetaMsg->numOfColumns, + .precision = pTableMetaMsg->precision + }; + + pTableMeta->sid = pTableMetaMsg->sid; + pTableMeta->uid = pTableMetaMsg->uid; + pTableMeta->vgid = pTableMetaMsg->vgid; + + pTableMeta->numOfVpeers = pTableMetaMsg->numOfVpeers; + memcpy(pTableMeta->vpeerDesc, pTableMetaMsg->vpeerDesc, sizeof(SVnodeDesc) * pTableMeta->numOfVpeers); + memcpy(pTableMeta->schema, pTableMetaMsg->schema, schemaSize); + + int32_t numOfTotalCols = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; + for(int32_t i = 0; i < numOfTotalCols; ++i) { + pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; + } + + if (size != NULL) { + *size = sizeof(STableMeta) + schemaSize; + } + + return pTableMeta; } + /** - * the MeterMeta data format in memory is as follows: + * the TableMeta data format in memory is as follows: * * +--------------------+ * |STableMeta Body data| sizeof(STableMeta) @@ -101,33 +195,15 @@ struct SSchema tsGetTbnameColumnSchema() { * |Tags data | tag_col_1.bytes + tag_col_2.bytes + .... * +--------------------+ * - * @param pMeta + * @param pTableMeta * @return */ -char* tsGetTagsValue(STableMeta* pMeta) { - int32_t numOfTotalCols = pMeta->numOfColumns + pMeta->numOfTags; - uint32_t offset = sizeof(STableMeta) + numOfTotalCols * sizeof(SSchema); - - return ((char*)pMeta + offset); -} +char* tsGetTagsValue(STableMeta* pTableMeta) { + int32_t offset = 0; +// int32_t numOfTotalCols = pTableMeta->numOfColumns + pTableMeta->numOfTags; +// uint32_t offset = sizeof(STableMeta) + numOfTotalCols * sizeof(SSchema); -bool tsMeterMetaIdentical(STableMeta* p1, STableMeta* p2) { - if (p1 == NULL || p2 == NULL || p1->uid != p2->uid || p1->sversion != p2->sversion) { - return false; - } - - if (p1 == p2) { - return true; - } - - size_t size = sizeof(STableMeta) + p1->numOfColumns * sizeof(SSchema); - - for (int32_t i = 0; i < p1->numOfTags; ++i) { - SSchema* pColSchema = tsGetColumnSchema(p1, i + p1->numOfColumns); - size += pColSchema->bytes; - } - - return memcmp(p1, p2, size) == 0; + return ((char*)pTableMeta + offset); } // todo refactor @@ -149,24 +225,6 @@ static FORCE_INLINE size_t copy(char* dst, const char* src, char delimiter) { return len; } -/** - * extract table name from meterid, which the format of userid.dbname.metername - * @param tableId - * @return - */ -void extractTableName(char* tableId, char* name) { - char* r = skipSegments(tableId, TS_PATH_DELIMITER[0], 2); - copy(name, r, TS_PATH_DELIMITER[0]); -} - -SSQLToken extractDBName(char* tableId, char* name) { - char* r = skipSegments(tableId, TS_PATH_DELIMITER[0], 1); - size_t len = copy(name, r, TS_PATH_DELIMITER[0]); - - SSQLToken token = {.z = name, .n = len, .type = TK_STRING}; - return token; -} - /* * tablePrefix.columnName * extract table name and save it in pTable, with only column name in pToken diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c index 51a59005f001f7555a3ffc2e5c7e15ed111af988..0ee8afd53c59969f11bfed85c1c32804e48050f9 100644 --- a/src/client/src/tscSecondaryMerge.c +++ b/src/client/src/tscSecondaryMerge.c @@ -319,12 +319,13 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd pRes->pLocalReducer = pReducer; pRes->numOfGroups = 0; - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); - int16_t prec = pMeterMetaInfo->pMeterMeta->precision; - + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); + + int16_t prec = tinfo.precision; int64_t stime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->stime : pQueryInfo->etime; int64_t revisedSTime = - taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->intervalTimeUnit, prec); + taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, prec); SInterpolationInfo *pInterpoInfo = &pReducer->interpolationInfo; taosInitInterpoInfo(pInterpoInfo, pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols, @@ -602,9 +603,9 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr *pFinalModel = NULL; SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * pMeterMetaInfo->pMetricMeta->numOfVnodes); + (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * pTableMetaInfo->pMetricMeta->numOfVnodes); if (*pMemBuffer == NULL) { tscError("%p failed to allocate memory", pSql); pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; @@ -635,7 +636,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr pModel = createColumnModel(pSchema, pQueryInfo->exprsInfo.numOfExprs, capacity); - for (int32_t i = 0; i < pMeterMetaInfo->pMetricMeta->numOfVnodes; ++i) { + for (int32_t i = 0; i < pTableMetaInfo->pMetricMeta->numOfVnodes; ++i) { (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pModel); (*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL; } @@ -650,7 +651,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); - SSchema *p1 = tsGetColumnSchema(pMeterMetaInfo->pMeterMeta, pExpr->colInfo.colIdx); + SSchema *p1 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIdx); int16_t inter = 0; int16_t type = -1; @@ -774,12 +775,14 @@ void adjustLoserTreeFromNewData(SLocalReducer *pLocalReducer, SLocalDataSource * void savePrevRecordAndSetupInterpoInfo(SLocalReducer *pLocalReducer, SQueryInfo *pQueryInfo, SInterpolationInfo *pInterpoInfo) { // discard following dataset in the same group and reset the interpolation information - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - int16_t prec = pMeterMetaInfo->pMeterMeta->precision; - + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); + + int16_t prec = tinfo.precision; int64_t stime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->stime : pQueryInfo->etime; int64_t revisedSTime = - taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->intervalTimeUnit, prec); + taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, prec); taosInitInterpoInfo(pInterpoInfo, pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols, pLocalReducer->rowSize); @@ -917,13 +920,15 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo functions[i] = tscSqlExprGet(pQueryInfo, i)->functionId; } - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); - int8_t precision = pMeterMetaInfo->pMeterMeta->precision; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); + + int8_t precision = tinfo.precision; while (1) { int32_t remains = taosNumOfRemainPoints(pInterpoInfo); TSKEY etime = taosGetRevisedEndKey(actualETime, pQueryInfo->order.order, pQueryInfo->intervalTime, - pQueryInfo->intervalTimeUnit, precision); + pQueryInfo->slidingTimeUnit, precision); int32_t nrows = taosGetNumOfResultWithInterpo(pInterpoInfo, pPrimaryKeys, remains, pQueryInfo->intervalTime, etime, pLocalReducer->resColModel->capacity); @@ -1268,14 +1273,16 @@ static void resetEnvForNewResultset(SSqlRes *pRes, SSqlCmd *pCmd, SLocalReducer pQueryInfo->limit.offset = pLocalReducer->offset; - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); - int16_t precision = pMeterMetaInfo->pMeterMeta->precision; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); + + int8_t precision = tinfo.precision; // for group result interpolation, do not return if not data is generated if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) { int64_t stime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->stime : pQueryInfo->etime; int64_t newTime = - taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->intervalTimeUnit, precision); + taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, precision); taosInitInterpoInfo(&pLocalReducer->interpolationInfo, pQueryInfo->order.order, newTime, pQueryInfo->groupbyExpr.numOfGroupCols, pLocalReducer->rowSize); @@ -1294,8 +1301,10 @@ static bool doInterpolationForCurrentGroup(SSqlObj *pSql) { SLocalReducer * pLocalReducer = pRes->pLocalReducer; SInterpolationInfo *pInterpoInfo = &pLocalReducer->interpolationInfo; - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - int8_t p = pMeterMetaInfo->pMeterMeta->precision; + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); + + int8_t p = tinfo.precision; if (taosHasRemainsDataForInterpolation(pInterpoInfo)) { assert(pQueryInfo->interpoType != TSDB_INTERPO_NONE); @@ -1305,7 +1314,7 @@ static bool doInterpolationForCurrentGroup(SSqlObj *pSql) { int32_t remain = taosNumOfRemainPoints(pInterpoInfo); TSKEY ekey = - taosGetRevisedEndKey(etime, pQueryInfo->order.order, pQueryInfo->intervalTime, pQueryInfo->intervalTimeUnit, p); + taosGetRevisedEndKey(etime, pQueryInfo->order.order, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, p); int32_t rows = taosGetNumOfResultWithInterpo(pInterpoInfo, (TSKEY *)pLocalReducer->pBufForInterpo, remain, pQueryInfo->intervalTime, ekey, pLocalReducer->resColModel->capacity); if (rows > 0) { // do interpo @@ -1328,8 +1337,10 @@ static bool doHandleLastRemainData(SSqlObj *pSql) { bool prevGroupCompleted = (!pLocalReducer->discard) && pLocalReducer->hasUnprocessedRow; SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - int8_t precision = pMeterMetaInfo->pMeterMeta->precision; + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); + int8_t precision = tinfo.precision; if ((isAllSourcesCompleted(pLocalReducer) && !pLocalReducer->hasPrevRow) || pLocalReducer->pLocalDataSrc[0] == NULL || prevGroupCompleted) { @@ -1338,7 +1349,7 @@ static bool doHandleLastRemainData(SSqlObj *pSql) { int64_t etime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->etime : pQueryInfo->stime; etime = taosGetRevisedEndKey(etime, pQueryInfo->order.order, pQueryInfo->intervalTime, - pQueryInfo->intervalTimeUnit, precision); + pQueryInfo->slidingTimeUnit, precision); int32_t rows = taosGetNumOfResultWithInterpo(pInterpoInfo, NULL, 0, pQueryInfo->intervalTime, etime, pLocalReducer->resColModel->capacity); if (rows > 0) { // do interpo diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index cc416e1cb7c358727ce75faedc40c682834f3ec7..7dd892550974ab37b6685e60ca2be29be06bfed2 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -16,13 +16,12 @@ #include "os.h" #include "tcache.h" #include "trpc.h" -#include "tscJoinProcess.h" #include "tscProfile.h" #include "tscSecondaryMerge.h" +#include "tscSubquery.h" #include "tscUtil.h" #include "tschemautil.h" #include "tsclient.h" -#include "tscompression.h" #include "tsocket.h" #include "ttime.h" #include "ttimer.h" @@ -30,9 +29,6 @@ #define TSC_MGMT_VNODE 999 -int tsMasterIndex = 0; -int tsSlaveIndex = 1; - SRpcIpSet tscMgmtIpList; SRpcIpSet tscDnodeIpSet; @@ -236,7 +232,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { if (rpcMsg->pCont == NULL) { rpcMsg->code = TSDB_CODE_NETWORK_UNAVAIL; } else { - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); if (rpcMsg->code == TSDB_CODE_NOT_ACTIVE_TABLE || rpcMsg->code == TSDB_CODE_INVALID_TABLE_ID || rpcMsg->code == TSDB_CODE_INVALID_VNODE_ID || rpcMsg->code == TSDB_CODE_NOT_ACTIVE_VNODE || rpcMsg->code == TSDB_CODE_NETWORK_UNAVAIL || rpcMsg->code == TSDB_CODE_NOT_ACTIVE_SESSION || @@ -265,9 +261,9 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { pSql->maxRetry = TSDB_VNODES_SUPPORT * 2; pSql->res.code = rpcMsg->code; // keep the previous error code - rpcMsg->code = tscRenewMeterMeta(pSql, pMeterMetaInfo->name); + rpcMsg->code = tscRenewMeterMeta(pSql, pTableMetaInfo->name); - if (pMeterMetaInfo->pMeterMeta) { + if (pTableMetaInfo->pTableMeta) { tscSendMsgToServer(pSql); rpcFreeCont(rpcMsg->pCont); return; @@ -278,8 +274,6 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { pSql->retry = 0; - if (pSql->fp == NULL) tsem_wait(&pSql->emptyRspSem); - pRes->rspLen = 0; if (pRes->code != TSDB_CODE_QUERY_CANCELLED) { pRes->code = (rpcMsg->code != TSDB_CODE_SUCCESS) ? rpcMsg->code : TSDB_CODE_NETWORK_UNAVAIL; @@ -328,187 +322,68 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { } } - if (pSql->fp == NULL) { - tsem_post(&pSql->rspSem); - } else { - if (pRes->code == TSDB_CODE_SUCCESS && tscProcessMsgRsp[pCmd->command]) - rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql); - - if (rpcMsg->code != TSDB_CODE_ACTION_IN_PROGRESS) { - int command = pCmd->command; - void *taosres = tscKeepConn[command] ? pSql : NULL; - rpcMsg->code = pRes->code ? -pRes->code : pRes->numOfRows; - - tscTrace("%p Async SQL result:%d res:%p", pSql, rpcMsg->code, taosres); - - /* - * Whether to free sqlObj or not should be decided before call the user defined function, since this SqlObj - * may be freed in UDF, and reused by other threads before tscShouldFreeAsyncSqlObj called, in which case - * tscShouldFreeAsyncSqlObj checks an object which is actually allocated by other threads. - * - * If this block of memory is re-allocated for an insert thread, in which tscKeepConn[command] equals to 0, - * the tscShouldFreeAsyncSqlObj will success and tscFreeSqlObj free it immediately. - */ - bool shouldFree = tscShouldFreeAsyncSqlObj(pSql); - if (command == TSDB_SQL_INSERT) { // handle multi-vnode insertion situation - (*pSql->fp)(pSql, taosres, rpcMsg->code); - } else { - (*pSql->fp)(pSql->param, taosres, rpcMsg->code); - } - - if (shouldFree) { - // If it is failed, all objects allocated during execution taos_connect_a should be released - if (command == TSDB_SQL_CONNECT) { - taos_close(pObj); - tscTrace("%p Async sql close failed connection", pSql); - } else { - tscFreeSqlObj(pSql); - tscTrace("%p Async sql is automatically freed", pSql); - } - } - } - } - - rpcFreeCont(rpcMsg->pCont); -} - -static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj); -static int tscLaunchSTableSubqueries(SSqlObj *pSql); + if (pRes->code == TSDB_CODE_SUCCESS && tscProcessMsgRsp[pCmd->command]) + rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql); -// todo merge with callback -int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSubquerySupporter *pSupporter) { - SSqlCmd * pCmd = &pSql->cmd; - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + if (rpcMsg->code != TSDB_CODE_ACTION_IN_PROGRESS) { + int command = pCmd->command; + void *taosres = tscKeepConn[command] ? pSql : NULL; + rpcMsg->code = pRes->code ? -pRes->code : pRes->numOfRows; - pSql->res.qhandle = 0x1; - pSql->res.numOfRows = 0; + tscTrace("%p Async SQL result:%d res:%p", pSql, rpcMsg->code, taosres); - if (pSql->pSubs == NULL) { - pSql->pSubs = calloc(pSupporter->pState->numOfTotal, POINTER_BYTES); - if (pSql->pSubs == NULL) { - return TSDB_CODE_CLI_OUT_OF_MEMORY; + /* + * Whether to free sqlObj or not should be decided before call the user defined function, since this SqlObj + * may be freed in UDF, and reused by other threads before tscShouldFreeAsyncSqlObj called, in which case + * tscShouldFreeAsyncSqlObj checks an object which is actually allocated by other threads. + * + * If this block of memory is re-allocated for an insert thread, in which tscKeepConn[command] equals to 0, + * the tscShouldFreeAsyncSqlObj will success and tscFreeSqlObj free it immediately. + */ + bool shouldFree = tscShouldFreeAsyncSqlObj(pSql); + if (command == TSDB_SQL_INSERT) { // handle multi-vnode insertion situation + (*pSql->fp)(pSql, taosres, rpcMsg->code); + } else { + (*pSql->fp)(pSql->param, taosres, rpcMsg->code); } - } - - SSqlObj *pNew = createSubqueryObj(pSql, tableIndex, tscJoinQueryCallback, pSupporter, NULL); - if (pNew == NULL) { - return TSDB_CODE_CLI_OUT_OF_MEMORY; - } - - pSql->pSubs[pSql->numOfSubs++] = pNew; - assert(pSql->numOfSubs <= pSupporter->pState->numOfTotal); - - if (QUERY_IS_JOIN_QUERY(pQueryInfo->type)) { - addGroupInfoForSubquery(pSql, pNew, 0, tableIndex); - // refactor as one method - SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); - assert(pNewQueryInfo != NULL); - - tscColumnBaseInfoUpdateTableIndex(&pNewQueryInfo->colList, 0); - tscColumnBaseInfoCopy(&pSupporter->colList, &pNewQueryInfo->colList, 0); - - tscSqlExprCopy(&pSupporter->exprsInfo, &pNewQueryInfo->exprsInfo, pSupporter->uid, false); - tscFieldInfoCopyAll(&pSupporter->fieldsInfo, &pNewQueryInfo->fieldsInfo); - - tscTagCondCopy(&pSupporter->tagCond, &pNewQueryInfo->tagCond); - - pNew->cmd.numOfCols = 0; - pNewQueryInfo->intervalTime = 0; - memset(&pNewQueryInfo->limit, 0, sizeof(SLimitVal)); - - // backup the data and clear it in the sqlcmd object - pSupporter->groupbyExpr = pNewQueryInfo->groupbyExpr; - memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SSqlGroupbyExpr)); - - // this data needs to be transfer to support struct - pNewQueryInfo->fieldsInfo.numOfOutputCols = 0; - pNewQueryInfo->exprsInfo.numOfExprs = 0; - - // set the ts,tags that involved in join, as the output column of intermediate result - tscClearSubqueryInfo(&pNew->cmd); - - SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = 1}; - SColumnIndex index = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - - tscAddSpecialColumnForSelect(pNewQueryInfo, 0, TSDB_FUNC_TS_COMP, &index, &colSchema, TSDB_COL_NORMAL); - - // set the tags value for ts_comp function - SSqlExpr *pExpr = tscSqlExprGet(pNewQueryInfo, 0); - - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pNewQueryInfo, 0); - int16_t tagColIndex = tscGetJoinTagColIndexByUid(&pSupporter->tagCond, pMeterMetaInfo->pMeterMeta->uid); - - pExpr->param->i64Key = tagColIndex; - pExpr->numOfParams = 1; - - // add the filter tag column - for (int32_t i = 0; i < pSupporter->colList.numOfCols; ++i) { - SColumnBase *pColBase = &pSupporter->colList.pColList[i]; - if (pColBase->numOfFilters > 0) { // copy to the pNew->cmd.colList if it is filtered. - tscColumnBaseCopy(&pNewQueryInfo->colList.pColList[pNewQueryInfo->colList.numOfCols], pColBase); - pNewQueryInfo->colList.numOfCols++; + if (shouldFree) { + // If it is failed, all objects allocated during execution taos_connect_a should be released + if (command == TSDB_SQL_CONNECT) { + taos_close(pObj); + tscTrace("%p Async sql close failed connection", pSql); + } else { + tscFreeSqlObj(pSql); + tscTrace("%p Async sql is automatically freed", pSql); } } - - tscTrace("%p subquery:%p tableIndex:%d, vnodeIdx:%d, type:%d, transfer to ts_comp query to retrieve timestamps, " - "exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s", - pSql, pNew, tableIndex, pMeterMetaInfo->vnodeIndex, pNewQueryInfo->type, - pNewQueryInfo->exprsInfo.numOfExprs, pNewQueryInfo->colList.numOfCols, - pNewQueryInfo->fieldsInfo.numOfOutputCols, pNewQueryInfo->pMeterInfo[0]->name); - tscPrintSelectClause(pNew, 0); - - tscTrace("%p subquery:%p tableIndex:%d, vnodeIdx:%d, type:%d, transfer to ts_comp query to retrieve timestamps, " - "exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s", - pSql, pNew, tableIndex, pMeterMetaInfo->vnodeIndex, pNewQueryInfo->type, - pNewQueryInfo->exprsInfo.numOfExprs, pNewQueryInfo->colList.numOfCols, - pNewQueryInfo->fieldsInfo.numOfOutputCols, pNewQueryInfo->pMeterInfo[0]->name); - tscPrintSelectClause(pNew, 0); - } else { - SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); - pNewQueryInfo->type |= TSDB_QUERY_TYPE_SUBQUERY; } -#ifdef _DEBUG_VIEW - tscPrintSelectClause(pNew, 0); -#endif - - return tscProcessSql(pNew); + rpcFreeCont(rpcMsg->pCont); } int doProcessSql(SSqlObj *pSql) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - void *asyncFp = pSql->fp; - if (pCmd->command == TSDB_SQL_SELECT || pCmd->command == TSDB_SQL_FETCH || pCmd->command == TSDB_SQL_RETRIEVE || - pCmd->command == TSDB_SQL_INSERT || pCmd->command == TSDB_SQL_CONNECT || pCmd->command == TSDB_SQL_HB || - pCmd->command == TSDB_SQL_META || pCmd->command == TSDB_SQL_METRIC) { + if (pCmd->command == TSDB_SQL_SELECT || + pCmd->command == TSDB_SQL_FETCH || + pCmd->command == TSDB_SQL_RETRIEVE || + pCmd->command == TSDB_SQL_INSERT || + pCmd->command == TSDB_SQL_CONNECT || + pCmd->command == TSDB_SQL_HB || + pCmd->command == TSDB_SQL_META || + pCmd->command == TSDB_SQL_METRIC) { tscBuildMsg[pCmd->command](pSql, NULL); } int32_t code = tscSendMsgToServer(pSql); - - if (asyncFp) { - if (code != TSDB_CODE_SUCCESS) { - pRes->code = code; - tscQueueAsyncRes(pSql); - } - return 0; - } - if (code != TSDB_CODE_SUCCESS) { pRes->code = code; - return code; + tscQueueAsyncRes(pSql); } - - tsem_wait(&pSql->rspSem); - - if (pRes->code == TSDB_CODE_SUCCESS && tscProcessMsgRsp[pCmd->command]) (*tscProcessMsgRsp[pCmd->command])(pSql); - - tsem_post(&pSql->emptyRspSem); - - return pRes->code; + + return TSDB_CODE_SUCCESS; } int tscProcessSql(SSqlObj *pSql) { @@ -517,13 +392,13 @@ int tscProcessSql(SSqlObj *pSql) { SSqlCmd *pCmd = &pSql->cmd; SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - SMeterMetaInfo *pMeterMetaInfo = NULL; - int16_t type = 0; + STableMetaInfo *pTableMetaInfo = NULL; + uint16_t type = 0; if (pQueryInfo != NULL) { - pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - if (pMeterMetaInfo != NULL) { - name = pMeterMetaInfo->name; + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + if (pTableMetaInfo != NULL) { + name = pTableMetaInfo->name; } type = pQueryInfo->type; @@ -534,21 +409,21 @@ int tscProcessSql(SSqlObj *pSql) { tscTrace("%p SQL cmd:%d will be processed, name:%s, type:%d", pSql, pCmd->command, name, type); if (pSql->cmd.command < TSDB_SQL_MGMT) { - // the pMeterMetaInfo cannot be NULL - if (pMeterMetaInfo == NULL) { + // the pTableMetaInfo cannot be NULL + if (pTableMetaInfo == NULL) { pSql->res.code = TSDB_CODE_OTHERS; return pSql->res.code; } // temp pSql->ipList = &tscMgmtIpList; -// if (UTIL_METER_IS_NOMRAL_METER(pMeterMetaInfo)) { -// pSql->index = pMeterMetaInfo->pMeterMeta->index; +// if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { +// pSql->index = pTableMetaInfo->pTableMeta->index; // } else { // it must be the parent SSqlObj for super table query // if ((pQueryInfo->type & TSDB_QUERY_TYPE_SUBQUERY) != 0) { -// int32_t idx = pMeterMetaInfo->vnodeIndex; +// int32_t idx = pTableMetaInfo->vnodeIndex; // -// SVnodeSidList *pSidList = tscGetVnodeSidList(pMeterMetaInfo->pMetricMeta, idx); +// SVnodeSidList *pSidList = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx); // pSql->index = pSidList->index; // } // } @@ -561,42 +436,7 @@ int tscProcessSql(SSqlObj *pSql) { // todo handle async situation if (QUERY_IS_JOIN_QUERY(type)) { if ((pQueryInfo->type & TSDB_QUERY_TYPE_SUBQUERY) == 0) { - SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); - - pState->numOfTotal = pQueryInfo->numOfTables; - - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - SJoinSubquerySupporter *pSupporter = tscCreateJoinSupporter(pSql, pState, i); - - if (pSupporter == NULL) { // failed to create support struct, abort current query - tscError("%p tableIndex:%d, failed to allocate join support object, abort further query", pSql, i); - pState->numOfCompleted = pQueryInfo->numOfTables - i - 1; - pSql->res.code = TSDB_CODE_CLI_OUT_OF_MEMORY; - - return pSql->res.code; - } - - int32_t code = tscLaunchJoinSubquery(pSql, i, pSupporter); - if (code != TSDB_CODE_SUCCESS) { // failed to create subquery object, quit query - tscDestroyJoinSupporter(pSupporter); - pSql->res.code = TSDB_CODE_CLI_OUT_OF_MEMORY; - - break; - } - } - - tsem_post(&pSql->emptyRspSem); - tsem_wait(&pSql->rspSem); - - tsem_post(&pSql->emptyRspSem); - - if (pSql->numOfSubs <= 0) { - pSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; - } else { - pSql->cmd.command = TSDB_SQL_METRIC_JOIN_RETRIEVE; - } - - return TSDB_CODE_SUCCESS; + return tscHandleMasterJoinQuery(pSql); } else { // for first stage sub query, iterate all vnodes to get all timestamp if ((pQueryInfo->type & TSDB_QUERY_TYPE_JOIN_SEC_STAGE) != TSDB_QUERY_TYPE_JOIN_SEC_STAGE) { @@ -605,461 +445,22 @@ int tscProcessSql(SSqlObj *pSql) { } } - if (tscIsTwoStageMergeMetricQuery(pQueryInfo, 0)) { - /* - * (ref. line: 964) - * Before this function returns from tscLaunchSTableSubqueries and continues, pSql may have been released at user - * program context after retrieving all data from vnodes. User function is called at tscRetrieveFromVnodeCallBack. - * - * when pSql being released, pSql->fp == NULL, it may pass the check of pSql->fp == NULL, - * which causes deadlock. So we keep it as local variable. - */ - if (tscLaunchSTableSubqueries(pSql) != TSDB_CODE_SUCCESS) { - return pRes->code; - } - - return pSql->res.code; - } else if (pSql->fp == (void(*)())launchMultivnodeInsert) { // multi-vnodes insertion - launchMultivnodeInsert(pSql); + if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { // super table query + tscHandleMasterSTableQuery(pSql); + return pRes->code; + } else if (pSql->fp == (void(*)())tscHandleMultivnodeInsert) { // multi-vnodes insertion + tscHandleMultivnodeInsert(pSql); return pSql->res.code; } return doProcessSql(pSql); } -static void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs, SSubqueryState* pState) { - assert(numOfSubs <= pSql->numOfSubs && numOfSubs >= 0 && pState != NULL); - - for(int32_t i = 0; i < numOfSubs; ++i) { - SSqlObj* pSub = pSql->pSubs[i]; - assert(pSub != NULL); - - SRetrieveSupport* pSupport = pSub->param; - - tfree(pSupport->localBuffer); - - pthread_mutex_unlock(&pSupport->queryMutex); - pthread_mutex_destroy(&pSupport->queryMutex); - - tfree(pSupport); - - tscFreeSqlObj(pSub); - } - - free(pState); -} - -int tscLaunchSTableSubqueries(SSqlObj *pSql) { - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - // pRes->code check only serves in launching metric sub-queries - if (pRes->code == TSDB_CODE_QUERY_CANCELLED) { - pCmd->command = TSDB_SQL_RETRIEVE_METRIC; // enable the abort of kill metric function. - return pRes->code; - } - - tExtMemBuffer ** pMemoryBuf = NULL; - tOrderDescriptor *pDesc = NULL; - SColumnModel * pModel = NULL; - - pRes->qhandle = 1; // hack the qhandle check - - const uint32_t nBufferSize = (1 << 16); // 64KB - - SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - int32_t numOfSubQueries = pMeterMetaInfo->pMetricMeta->numOfVnodes; - assert(numOfSubQueries > 0); - - int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, nBufferSize); - if (ret != 0) { - pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; - if (pSql->fp) { - tscQueueAsyncRes(pSql); - } - return pRes->code; - } - - pSql->pSubs = calloc(numOfSubQueries, POINTER_BYTES); - pSql->numOfSubs = numOfSubQueries; - - tscTrace("%p retrieved query data from %d vnode(s)", pSql, numOfSubQueries); - SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); - pState->numOfTotal = numOfSubQueries; - pRes->code = TSDB_CODE_SUCCESS; - - int32_t i = 0; - for (; i < numOfSubQueries; ++i) { - SRetrieveSupport *trs = (SRetrieveSupport *)calloc(1, sizeof(SRetrieveSupport)); - if (trs == NULL) { - tscError("%p failed to malloc buffer for SRetrieveSupport, orderOfSub:%d, reason:%s", pSql, i, strerror(errno)); - break; - } - - trs->pExtMemBuffer = pMemoryBuf; - trs->pOrderDescriptor = pDesc; - trs->pState = pState; - - trs->localBuffer = (tFilePage *)calloc(1, nBufferSize + sizeof(tFilePage)); - if (trs->localBuffer == NULL) { - tscError("%p failed to malloc buffer for local buffer, orderOfSub:%d, reason:%s", pSql, i, strerror(errno)); - tfree(trs); - break; - } - - trs->subqueryIndex = i; - trs->pParentSqlObj = pSql; - trs->pFinalColModel = pModel; - - pthread_mutexattr_t mutexattr = {0}; - pthread_mutexattr_settype(&mutexattr, PTHREAD_MUTEX_RECURSIVE_NP); - pthread_mutex_init(&trs->queryMutex, &mutexattr); - pthread_mutexattr_destroy(&mutexattr); - - SSqlObj *pNew = tscCreateSqlObjForSubquery(pSql, trs, NULL); - if (pNew == NULL) { - tscError("%p failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql, i, strerror(errno)); - tfree(trs->localBuffer); - tfree(trs); - break; - } - - // todo handle multi-vnode situation - if (pQueryInfo->tsBuf) { - SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); - pNewQueryInfo->tsBuf = tsBufClone(pQueryInfo->tsBuf); - } - - tscTrace("%p sub:%p create subquery success. orderOfSub:%d", pSql, pNew, trs->subqueryIndex); - } - - if (i < numOfSubQueries) { - tscError("%p failed to prepare subquery structure and launch subqueries", pSql); - pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; - - tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, numOfSubQueries); - doCleanupSubqueries(pSql, i, pState); - return pRes->code; // free all allocated resource - } - - if (pRes->code == TSDB_CODE_QUERY_CANCELLED) { - tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, numOfSubQueries); - doCleanupSubqueries(pSql, i, pState); - return pRes->code; - } - - for(int32_t j = 0; j < numOfSubQueries; ++j) { - SSqlObj* pSub = pSql->pSubs[j]; - SRetrieveSupport* pSupport = pSub->param; - - tscTrace("%p sub:%p launch subquery, orderOfSub:%d.", pSql, pSub, pSupport->subqueryIndex); - tscProcessSql(pSub); - } - - return TSDB_CODE_SUCCESS; -} - -static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) { - tscTrace("%p start to free subquery result", pSql); - - if (pSql->res.code == TSDB_CODE_SUCCESS) { - taos_free_result(pSql); - } - - tfree(trsupport->localBuffer); - - pthread_mutex_unlock(&trsupport->queryMutex); - pthread_mutex_destroy(&trsupport->queryMutex); - - tfree(trsupport); -} - -static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows); - -static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES *tres, int32_t errCode) { -// set no disk space error info -#ifdef WINDOWS - LPVOID lpMsgBuf; - FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, - GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), // Default language - (LPTSTR)&lpMsgBuf, 0, NULL); - tscError("sub:%p failed to flush data to disk:reason:%s", tres, lpMsgBuf); - LocalFree(lpMsgBuf); -#else - char buf[256] = {0}; - strerror_r(errno, buf, 256); - tscError("sub:%p failed to flush data to disk:reason:%s", tres, buf); -#endif - - trsupport->pState->code = -errCode; - trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - - pthread_mutex_unlock(&trsupport->queryMutex); - - tscRetrieveFromVnodeCallBack(trsupport, tres, trsupport->pState->code); -} - -static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows) { - SSqlObj *pPObj = trsupport->pParentSqlObj; - int32_t subqueryIndex = trsupport->subqueryIndex; - - assert(pSql != NULL); - SSubqueryState* pState = trsupport->pState; - assert(pState->numOfCompleted < pState->numOfTotal && pState->numOfCompleted >= 0 && - pPObj->numOfSubs == pState->numOfTotal); - - /* retrieved in subquery failed. OR query cancelled in retrieve phase. */ - if (pState->code == TSDB_CODE_SUCCESS && pPObj->res.code != TSDB_CODE_SUCCESS) { - pState->code = -(int)pPObj->res.code; - - /* - * kill current sub-query connection, which may retrieve data from vnodes; - * Here we get: pPObj->res.code == TSDB_CODE_QUERY_CANCELLED - */ - pSql->res.numOfRows = 0; - trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; // disable retry efforts - tscTrace("%p query is cancelled, sub:%p, orderOfSub:%d abort retrieve, code:%d", trsupport->pParentSqlObj, pSql, - subqueryIndex, pState->code); - } - - if (numOfRows >= 0) { // current query is successful, but other sub query failed, still abort current query. - tscTrace("%p sub:%p retrieve numOfRows:%d,orderOfSub:%d", pPObj, pSql, numOfRows, subqueryIndex); - tscError("%p sub:%p abort further retrieval due to other queries failure,orderOfSub:%d,code:%d", pPObj, pSql, - subqueryIndex, pState->code); - } else { - if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY && pState->code == TSDB_CODE_SUCCESS) { - /* - * current query failed, and the retry count is less than the available - * count, retry query clear previous retrieved data, then launch a new sub query - */ - tExtMemBufferClear(trsupport->pExtMemBuffer[subqueryIndex]); - - // clear local saved number of results - trsupport->localBuffer->numOfElems = 0; - pthread_mutex_unlock(&trsupport->queryMutex); - - tscTrace("%p sub:%p retrieve failed, code:%d, orderOfSub:%d, retry:%d", trsupport->pParentSqlObj, pSql, numOfRows, - subqueryIndex, trsupport->numOfRetry); - - SSqlObj *pNew = tscCreateSqlObjForSubquery(trsupport->pParentSqlObj, trsupport, pSql); - if (pNew == NULL) { - tscError("%p sub:%p failed to create new subquery sqlobj due to out of memory, abort retry", - trsupport->pParentSqlObj, pSql); - - pState->code = TSDB_CODE_CLI_OUT_OF_MEMORY; - trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - return; - } - - tscProcessSql(pNew); - return; - } else { // reach the maximum retry count, abort - atomic_val_compare_exchange_32(&pState->code, TSDB_CODE_SUCCESS, numOfRows); - tscError("%p sub:%p retrieve failed,code:%d,orderOfSub:%d failed.no more retry,set global code:%d", pPObj, pSql, - numOfRows, subqueryIndex, pState->code); - } - } - - int32_t numOfTotal = pState->numOfTotal; - - int32_t finished = atomic_add_fetch_32(&pState->numOfCompleted, 1); - if (finished < numOfTotal) { - tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex, finished); - return tscFreeSubSqlObj(trsupport, pSql); - } - - // all subqueries are failed - tscError("%p retrieve from %d vnode(s) completed,code:%d.FAILED.", pPObj, pState->numOfTotal, pState->code); - pPObj->res.code = -(pState->code); - - // release allocated resource - tscLocalReducerEnvDestroy(trsupport->pExtMemBuffer, trsupport->pOrderDescriptor, trsupport->pFinalColModel, - pState->numOfTotal); - - tfree(trsupport->pState); - tscFreeSubSqlObj(trsupport, pSql); - - // sync query, wait for the master SSqlObj to proceed - if (pPObj->fp == NULL) { - // sync query, wait for the master SSqlObj to proceed - tsem_wait(&pPObj->emptyRspSem); - tsem_wait(&pPObj->emptyRspSem); - - tsem_post(&pPObj->rspSem); - - pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC; - } else { - // in case of second stage join subquery, invoke its callback function instead of regular QueueAsyncRes - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pPObj->cmd, 0); - - if ((pQueryInfo->type & TSDB_QUERY_TYPE_JOIN_SEC_STAGE) == TSDB_QUERY_TYPE_JOIN_SEC_STAGE) { - (*pPObj->fp)(pPObj->param, pPObj, pPObj->res.code); - } else { // regular super table query - if (pPObj->res.code != TSDB_CODE_SUCCESS) { - tscQueueAsyncRes(pPObj); - } - } - } -} - -void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { - SRetrieveSupport *trsupport = (SRetrieveSupport *)param; - int32_t idx = trsupport->subqueryIndex; - SSqlObj * pPObj = trsupport->pParentSqlObj; - tOrderDescriptor *pDesc = trsupport->pOrderDescriptor; - - SSqlObj *pSql = (SSqlObj *)tres; - if (pSql == NULL) { // sql object has been released in error process, return immediately - tscTrace("%p subquery has been released, idx:%d, abort", pPObj, idx); - return; - } - - SSubqueryState* pState = trsupport->pState; - assert(pState->numOfCompleted < pState->numOfTotal && pState->numOfCompleted >= 0 && - pPObj->numOfSubs == pState->numOfTotal); - - // query process and cancel query process may execute at the same time - pthread_mutex_lock(&trsupport->queryMutex); - - if (numOfRows < 0 || pState->code < 0 || pPObj->res.code != TSDB_CODE_SUCCESS) { - return tscHandleSubRetrievalError(trsupport, pSql, numOfRows); - } - - SSqlRes * pRes = &pSql->res; - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - - SVnodeSidList *vnodeInfo = tscGetVnodeSidList(pMeterMetaInfo->pMetricMeta, idx); - SVnodeDesc * pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index]; - - if (numOfRows > 0) { - assert(pRes->numOfRows == numOfRows); - int64_t num = atomic_add_fetch_64(&pState->numOfRetrievedRows, numOfRows); - - tscTrace("%p sub:%p retrieve numOfRows:%d totalNumOfRows:%d from ip:%u,vid:%d,orderOfSub:%d", pPObj, pSql, - pRes->numOfRows, pState->numOfRetrievedRows, pSvd->ip, pSvd->vnode, idx); - - if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0)) { - tscError("%p sub:%p num of OrderedRes is too many, max allowed:%" PRId64 " , current:%" PRId64, - pPObj, pSql, tsMaxNumOfOrderedResults, num); - tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_SORTED_RES_TOO_MANY); - return; - } - - -#ifdef _DEBUG_VIEW - printf("received data from vnode: %d rows\n", pRes->numOfRows); - SSrcColumnInfo colInfo[256] = {0}; - - tscGetSrcColumnInfo(colInfo, pQueryInfo); - tColModelDisplayEx(pDesc->pColumnModel, pRes->data, pRes->numOfRows, pRes->numOfRows, colInfo); -#endif - if (tsTotalTmpDirGB != 0 && tsAvailTmpDirGB < tsMinimalTmpDirGB) { - tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pPObj, pSql, - tsAvailTmpDirGB, tsMinimalTmpDirGB); - tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE); - return; - } - - int32_t ret = saveToBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, pRes->data, - pRes->numOfRows, pQueryInfo->groupbyExpr.orderType); - if (ret < 0) { - // set no disk space error info, and abort retry - tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE); - } else { - pthread_mutex_unlock(&trsupport->queryMutex); - taos_fetch_rows_a(tres, tscRetrieveFromVnodeCallBack, param); - } - - } else { // all data has been retrieved to client - /* data in from current vnode is stored in cache and disk */ - uint32_t numOfRowsFromVnode = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->numOfElems; - tscTrace("%p sub:%p all data retrieved from ip:%u,vid:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql, pSvd->ip, - pSvd->vnode, numOfRowsFromVnode, idx); - - tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity); - -#ifdef _DEBUG_VIEW - printf("%" PRIu64 " rows data flushed to disk:\n", trsupport->localBuffer->numOfElems); - SSrcColumnInfo colInfo[256] = {0}; - tscGetSrcColumnInfo(colInfo, pQueryInfo); - tColModelDisplayEx(pDesc->pColumnModel, trsupport->localBuffer->data, trsupport->localBuffer->numOfElems, - trsupport->localBuffer->numOfElems, colInfo); -#endif - - if (tsTotalTmpDirGB != 0 && tsAvailTmpDirGB < tsMinimalTmpDirGB) { - tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pPObj, pSql, - tsAvailTmpDirGB, tsMinimalTmpDirGB); - tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE); - return; - } - - // each result for a vnode is ordered as an independant list, - // then used as an input of loser tree for disk-based merge routine - int32_t ret = tscFlushTmpBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, - pQueryInfo->groupbyExpr.orderType); - if (ret != 0) { - /* set no disk space error info, and abort retry */ - return tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE); - } - - // keep this value local variable, since the pState variable may be released by other threads, if atomic_add opertion - // increases the finished value up to pState->numOfTotal value, which means all subqueries are completed. - // In this case, the comparsion between finished value and released pState->numOfTotal is not safe. - int32_t numOfTotal = pState->numOfTotal; - - int32_t finished = atomic_add_fetch_32(&pState->numOfCompleted, 1); - if (finished < numOfTotal) { - tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex, finished); - return tscFreeSubSqlObj(trsupport, pSql); - } - - // all sub-queries are returned, start to local merge process - pDesc->pColumnModel->capacity = trsupport->pExtMemBuffer[idx]->numOfElemsPerPage; - - tscTrace("%p retrieve from %d vnodes completed.final NumOfRows:%d,start to build loser tree", pPObj, - pState->numOfTotal, pState->numOfRetrievedRows); - - SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pPObj->cmd, 0); - tscClearInterpInfo(pPQueryInfo); - - tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfTotal, pDesc, trsupport->pFinalColModel, - &pPObj->cmd, &pPObj->res); - tscTrace("%p build loser tree completed", pPObj); - - pPObj->res.precision = pSql->res.precision; - pPObj->res.numOfRows = 0; - pPObj->res.row = 0; - - // only free once - tfree(trsupport->pState); - - tscFreeSubSqlObj(trsupport, pSql); - - if (pPObj->fp == NULL) { - tsem_wait(&pPObj->emptyRspSem); - tsem_wait(&pPObj->emptyRspSem); - - tsem_post(&pPObj->rspSem); - } else { - // set the command flag must be after the semaphore been correctly set. - pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC; - if (pPObj->res.code == TSDB_CODE_SUCCESS) { - (*pPObj->fp)(pPObj->param, pPObj, 0); - } else { - tscQueueAsyncRes(pPObj); - } - } - } -} - void tscKillMetricQuery(SSqlObj *pSql) { SSqlCmd* pCmd = &pSql->cmd; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - if (!tscIsTwoStageMergeMetricQuery(pQueryInfo, 0)) { + if (!tscIsTwoStageSTableQuery(pQueryInfo, 0)) { return; } @@ -1097,117 +498,6 @@ void tscKillMetricQuery(SSqlObj *pSql) { tscTrace("%p metric query is cancelled", pSql); } -static void tscRetrieveDataRes(void *param, TAOS_RES *tres, int retCode); - -static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj) { - const int32_t table_index = 0; - - SSqlObj *pNew = createSubqueryObj(pSql, table_index, tscRetrieveDataRes, trsupport, prevSqlObj); - if (pNew != NULL) { // the sub query of two-stage super table query - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); - pQueryInfo->type |= TSDB_QUERY_TYPE_STABLE_SUBQUERY; - - assert(pQueryInfo->numOfTables == 1 && pNew->cmd.numOfClause == 1); - - // launch subquery for each vnode, so the subquery index equals to the vnodeIndex. - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, table_index); - pMeterMetaInfo->vnodeIndex = trsupport->subqueryIndex; - - pSql->pSubs[trsupport->subqueryIndex] = pNew; - } - - return pNew; -} - -void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { - SRetrieveSupport *trsupport = (SRetrieveSupport *)param; - - SSqlObj* pParentSql = trsupport->pParentSqlObj; - SSqlObj* pSql = (SSqlObj *)tres; - - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, 0, 0); - assert(pSql->cmd.numOfClause == 1 && pSql->cmd.pQueryInfo[0]->numOfTables == 1); - - int32_t idx = pMeterMetaInfo->vnodeIndex; - - SVnodeSidList *vnodeInfo = NULL; - SVnodeDesc * pSvd = NULL; - if (pMeterMetaInfo->pMetricMeta != NULL) { - vnodeInfo = tscGetVnodeSidList(pMeterMetaInfo->pMetricMeta, idx); - pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index]; - } - - SSubqueryState* pState = trsupport->pState; - assert(pState->numOfCompleted < pState->numOfTotal && pState->numOfCompleted >= 0 && - pParentSql->numOfSubs == pState->numOfTotal); - - if (pParentSql->res.code != TSDB_CODE_SUCCESS || pState->code != TSDB_CODE_SUCCESS) { - // metric query is killed, Note: code must be less than 0 - trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - code = -(int)(pParentSql->res.code); - } else { - code = pState->code; - } - tscTrace("%p query cancelled or failed, sub:%p, orderOfSub:%d abort, code:%d", pParentSql, pSql, - trsupport->subqueryIndex, code); - } - - /* - * if a query on a vnode is failed, all retrieve operations from vnode that occurs later - * than this one are actually not necessary, we simply call the tscRetrieveFromVnodeCallBack - * function to abort current and remain retrieve process. - * - * NOTE: threadsafe is required. - */ - if (code != TSDB_CODE_SUCCESS) { - if (trsupport->numOfRetry++ >= MAX_NUM_OF_SUBQUERY_RETRY) { - tscTrace("%p sub:%p reach the max retry count,set global code:%d", pParentSql, pSql, code); - atomic_val_compare_exchange_32(&pState->code, 0, code); - } else { // does not reach the maximum retry count, go on - tscTrace("%p sub:%p failed code:%d, retry:%d", pParentSql, pSql, code, trsupport->numOfRetry); - - SSqlObj *pNew = tscCreateSqlObjForSubquery(pParentSql, trsupport, pSql); - if (pNew == NULL) { - tscError("%p sub:%p failed to create new subquery due to out of memory, abort retry, vid:%d, orderOfSub:%d", - trsupport->pParentSqlObj, pSql, pSvd != NULL ? pSvd->vnode : -1, trsupport->subqueryIndex); - - pState->code = -TSDB_CODE_CLI_OUT_OF_MEMORY; - trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - } else { - SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); - assert(pNewQueryInfo->pMeterInfo[0]->pMeterMeta != NULL && pNewQueryInfo->pMeterInfo[0]->pMetricMeta != NULL); - tscProcessSql(pNew); - return; - } - } - } - - if (pState->code != TSDB_CODE_SUCCESS) { // failed, abort - if (vnodeInfo != NULL) { - tscTrace("%p sub:%p query failed,ip:%u,vid:%d,orderOfSub:%d,global code:%d", pParentSql, pSql, - vnodeInfo->vpeerDesc[vnodeInfo->index].ip, vnodeInfo->vpeerDesc[vnodeInfo->index].vnode, - trsupport->subqueryIndex, pState->code); - } else { - tscTrace("%p sub:%p query failed,orderOfSub:%d,global code:%d", pParentSql, pSql, - trsupport->subqueryIndex, pState->code); - } - - tscRetrieveFromVnodeCallBack(param, tres, pState->code); - } else { // success, proceed to retrieve data from dnode - if (vnodeInfo != NULL) { - tscTrace("%p sub:%p query complete,ip:%u,vid:%d,orderOfSub:%d,retrieve data", trsupport->pParentSqlObj, pSql, - vnodeInfo->vpeerDesc[vnodeInfo->index].ip, vnodeInfo->vpeerDesc[vnodeInfo->index].vnode, - trsupport->subqueryIndex); - } else { - tscTrace("%p sub:%p query complete, orderOfSub:%d,retrieve data", trsupport->pParentSqlObj, pSql, - trsupport->subqueryIndex); - } - - taos_fetch_rows_a(tres, tscRetrieveFromVnodeCallBack, param); - } -} - int tscBuildRetrieveMsg(SSqlObj *pSql, SSqlInfo *pInfo) { char *pMsg, *pStart; @@ -1231,16 +521,16 @@ int tscBuildRetrieveMsg(SSqlObj *pSql, SSqlInfo *pInfo) { void tscUpdateVnodeInSubmitMsg(SSqlObj *pSql, char *buf) { //SShellSubmitMsg *pShellMsg; //char * pMsg; - //SMeterMetaInfo * pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, pSql->cmd.clauseIndex, 0); + //STableMetaInfo * pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); - //STableMeta *pMeterMeta = pMeterMetaInfo->pMeterMeta; + //STableMeta *pTableMeta = pTableMetaInfo->pTableMeta; //pMsg = buf + tsRpcHeadSize; //TODO set iplist //pShellMsg = (SShellSubmitMsg *)pMsg; - //pShellMsg->vnode = htons(pMeterMeta->vpeerDesc[pSql->index].vnode); - //tscTrace("%p update submit msg vnode:%s:%d", pSql, taosIpStr(pMeterMeta->vpeerDesc[pSql->index].ip), + //pShellMsg->vnode = htons(pTableMeta->vpeerDesc[pSql->index].vnode); + //tscTrace("%p update submit msg vnode:%s:%d", pSql, taosIpStr(pTableMeta->vpeerDesc[pSql->index].ip), // htons(pShellMsg->vnode)); } @@ -1248,24 +538,21 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SShellSubmitMsg *pShellMsg; char * pMsg, *pStart; - SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - - STableMeta *pMeterMeta = pMeterMetaInfo->pMeterMeta; - + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); + pStart = pSql->cmd.payload + tsRpcHeadSize; pMsg = pStart; pShellMsg = (SShellSubmitMsg *)pMsg; pShellMsg->import = htons(TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT) ? 0 : 1); - pShellMsg->vnode = 0; //htons(pMeterMeta->vpeerDesc[pMeterMeta->index].vnode); + pShellMsg->vnode = 0; //htons(pTableMeta->vpeerDesc[pTableMeta->index].vnode); pShellMsg->numOfSid = htonl(pSql->cmd.numOfTablesInSubmit); // number of meters to be inserted // pSql->cmd.payloadLen is set during parse sql routine, so we do not use it here pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; - tscTrace("%p update submit msg vnode:%s:%d", pSql, taosIpStr(pMeterMeta->vpeerDesc[pMeterMeta->index].ip), - htons(pShellMsg->vnode)); +// tscTrace("%p update submit msg vnode:%s:%d", pSql, taosIpStr(pTableMeta->vpeerDesc[pTableMeta->index].ip), +// htons(pShellMsg->vnode)); pSql->cmd.payloadLen = sizeof(SShellSubmitMsg); @@ -1275,17 +562,17 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { void tscUpdateVnodeInQueryMsg(SSqlObj *pSql, char *buf) { //TODO // SSqlCmd * pCmd = &pSql->cmd; -// SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); +// STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); // // char * pStart = buf + tsRpcHeadSize; // SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pStart; // -// if (UTIL_METER_IS_NOMRAL_METER(pMeterMetaInfo)) { // pColumnModel == NULL, query on meter -// STableMeta *pMeterMeta = pMeterMetaInfo->pMeterMeta; -// pQueryMsg->vnode = htons(pMeterMeta->vpeerDesc[pSql->index].vnode); +// if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { // pColumnModel == NULL, query on meter +// STableMeta *pTableMeta = pTableMetaInfo->pTableMeta; +// pQueryMsg->vnode = htons(pTableMeta->vpeerDesc[pSql->index].vnode); // } else { // query on metric -// SSuperTableMeta * pMetricMeta = pMeterMetaInfo->pMetricMeta; -// SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pMeterMetaInfo->vnodeIndex); +// SSuperTableMeta * pMetricMeta = pTableMetaInfo->pMetricMeta; +// SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vnodeIndex); // pQueryMsg->vnode = htons(pVnodeSidList->vpeerDesc[pSql->index].vnode); // } } @@ -1301,15 +588,15 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { int32_t srcColListSize = pQueryInfo->colList.numOfCols * sizeof(SColumnInfo); int32_t exprSize = sizeof(SSqlFuncExprMsg) * pQueryInfo->exprsInfo.numOfExprs; - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); // meter query without tags values - if (!UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) { + if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize; } - SSuperTableMeta *pMetricMeta = pMeterMetaInfo->pMetricMeta; - SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pMeterMetaInfo->vnodeIndex); + SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta; + SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vnodeIndex); int32_t meterInfoSize = (pMetricMeta->tagLen + sizeof(STableSidExtInfo)) * pVnodeSidList->numOfSids; int32_t outputColumnSize = pQueryInfo->exprsInfo.numOfExprs * sizeof(SSqlFuncExprMsg); @@ -1323,31 +610,31 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { } static char *doSerializeTableInfo(SSqlObj *pSql, int32_t numOfTables, int32_t vnodeId, char *pMsg) { - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, pSql->cmd.clauseIndex, 0); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); - STableMeta * pMeterMeta = pMeterMetaInfo->pMeterMeta; - SSuperTableMeta *pMetricMeta = pMeterMetaInfo->pMetricMeta; + STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; + SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta; tscTrace("%p vid:%d, query on %d meters", pSql, vnodeId, numOfTables); - if (UTIL_METER_IS_NOMRAL_METER(pMeterMetaInfo)) { + if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { #ifdef _DEBUG_VIEW - tscTrace("%p sid:%d, uid:%" PRIu64, pSql, pMeterMetaInfo->pMeterMeta->sid, pMeterMetaInfo->pMeterMeta->uid); + tscTrace("%p sid:%d, uid:%" PRIu64, pSql, pTableMetaInfo->pTableMeta->sid, pTableMetaInfo->pTableMeta->uid); #endif - STableSidExtInfo *pMeterInfo = (STableSidExtInfo *)pMsg; - pMeterInfo->sid = htonl(pMeterMeta->sid); - pMeterInfo->uid = htobe64(pMeterMeta->uid); - pMeterInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pMeterMeta->uid)); + STableSidExtInfo *pTableMetaInfo = (STableSidExtInfo *)pMsg; + pTableMetaInfo->sid = htonl(pTableMeta->sid); + pTableMetaInfo->uid = htobe64(pTableMeta->uid); + pTableMetaInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid)); pMsg += sizeof(STableSidExtInfo); } else { - SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pMeterMetaInfo->vnodeIndex); + SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vnodeIndex); for (int32_t i = 0; i < numOfTables; ++i) { - STableSidExtInfo *pMeterInfo = (STableSidExtInfo *)pMsg; + STableSidExtInfo *pTableMetaInfo = (STableSidExtInfo *)pMsg; STableSidExtInfo *pQueryMeterInfo = tscGetMeterSidInfo(pVnodeSidList, i); - pMeterInfo->sid = htonl(pQueryMeterInfo->sid); - pMeterInfo->uid = htobe64(pQueryMeterInfo->uid); - pMeterInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pQueryMeterInfo->uid)); + pTableMetaInfo->sid = htonl(pQueryMeterInfo->sid); + pTableMetaInfo->uid = htobe64(pQueryMeterInfo->uid); + pTableMetaInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pQueryMeterInfo->uid)); pMsg += sizeof(STableSidExtInfo); @@ -1374,34 +661,32 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - char * pStart = pCmd->payload + tsRpcHeadSize; + char *pStart = pCmd->payload + tsRpcHeadSize; - STableMeta * pMeterMeta = pMeterMetaInfo->pMeterMeta; - SSuperTableMeta *pMetricMeta = pMeterMetaInfo->pMetricMeta; + STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; + SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta; SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pStart; int32_t msgLen = 0; int32_t numOfTables = 0; - if (UTIL_METER_IS_NOMRAL_METER(pMeterMetaInfo)) { + if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { numOfTables = 1; - tscTrace("%p query on vnode: %d, number of sid:%d, meter id: %s", pSql, - pMeterMeta->vpeerDesc[pMeterMeta->index].vnode, 1, pMeterMetaInfo->name); - - pQueryMsg->vnode = htons(pMeterMeta->vpeerDesc[pMeterMeta->index].vnode); - pQueryMsg->uid = pMeterMeta->uid; + pQueryMsg->uid = pTableMeta->uid; pQueryMsg->numOfTagsCols = 0; + + tscTrace("%p queried tables:%d, table id: %s", pSql, 1, pTableMetaInfo->name); } else { // query on super table - if (pMeterMetaInfo->vnodeIndex < 0) { - tscError("%p error vnodeIdx:%d", pSql, pMeterMetaInfo->vnodeIndex); + if (pTableMetaInfo->vnodeIndex < 0) { + tscError("%p error vnodeIdx:%d", pSql, pTableMetaInfo->vnodeIndex); return -1; } - SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pMeterMetaInfo->vnodeIndex); + SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vnodeIndex); uint32_t vnodeId = pVnodeSidList->vpeerDesc[pVnodeSidList->index].vnode; numOfTables = pVnodeSidList->numOfSids; @@ -1410,19 +695,19 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { return -1; // error } - tscTrace("%p query on vid:%d, number of sid:%d", pSql, vnodeId, numOfTables); + tscTrace("%p query on vid:%d, number of tables:%d", pSql, vnodeId, numOfTables); pQueryMsg->vnode = htons(vnodeId); } - pQueryMsg->numOfSids = htonl(numOfTables); - pQueryMsg->numOfTagsCols = htons(pMeterMetaInfo->numOfTags); + pQueryMsg->numOfTables = htonl(numOfTables); + pQueryMsg->numOfTagsCols = htons(pTableMetaInfo->numOfTags); if (pQueryInfo->order.order == TSQL_SO_ASC) { - pQueryMsg->skey = htobe64(pQueryInfo->stime); - pQueryMsg->ekey = htobe64(pQueryInfo->etime); + pQueryMsg->window.skey = htobe64(pQueryInfo->stime); + pQueryMsg->window.ekey = htobe64(pQueryInfo->etime); } else { - pQueryMsg->skey = htobe64(pQueryInfo->etime); - pQueryMsg->ekey = htobe64(pQueryInfo->stime); + pQueryMsg->window.skey = htobe64(pQueryInfo->etime); + pQueryMsg->window.ekey = htobe64(pQueryInfo->stime); } pQueryMsg->order = htons(pQueryInfo->order.order); @@ -1436,17 +721,12 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->numOfCols = htons(pQueryInfo->colList.numOfCols); if (pQueryInfo->colList.numOfCols <= 0) { - tscError("%p illegal value of numOfCols in query msg: %d", pSql, pMeterMeta->numOfColumns); - return -1; - } - - if (pMeterMeta->numOfTags < 0) { - tscError("%p illegal value of numOfTagsCols in query msg: %d", pSql, pMeterMeta->numOfTags); + tscError("%p illegal value of numOfCols in query msg: %d", pSql, tscGetNumOfColumns(pTableMeta)); return -1; } pQueryMsg->intervalTime = htobe64(pQueryInfo->intervalTime); - pQueryMsg->intervalTimeUnit = pQueryInfo->intervalTimeUnit; + pQueryMsg->slidingTimeUnit = pQueryInfo->slidingTimeUnit; pQueryMsg->slidingTime = htobe64(pQueryInfo->slidingTime); if (pQueryInfo->intervalTime < 0) { @@ -1461,9 +741,9 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); - if (UTIL_METER_IS_NOMRAL_METER(pMeterMetaInfo)) { // query on meter + if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { // query on meter pQueryMsg->tagLength = 0; - } else { // query on metric + } else { // query on super table pQueryMsg->tagLength = htons(pMetricMeta->tagLen); } @@ -1478,16 +758,16 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // set column list ids char * pMsg = (char *)(pQueryMsg->colList) + pQueryInfo->colList.numOfCols * sizeof(SColumnInfo); - SSchema *pSchema = tsGetSchema(pMeterMeta); + SSchema *pSchema = tscGetTableSchema(pTableMeta); for (int32_t i = 0; i < pQueryInfo->colList.numOfCols; ++i) { SColumnBase *pCol = tscColumnBaseInfoGet(&pQueryInfo->colList, i); SSchema * pColSchema = &pSchema[pCol->colIndex.columnIndex]; - if (pCol->colIndex.columnIndex >= pMeterMeta->numOfColumns || pColSchema->type < TSDB_DATA_TYPE_BOOL || + if (pCol->colIndex.columnIndex >= tscGetNumOfColumns(pTableMeta) || pColSchema->type < TSDB_DATA_TYPE_BOOL || pColSchema->type > TSDB_DATA_TYPE_NCHAR) { tscError("%p vid:%d sid:%d id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s", pSql, - htons(pQueryMsg->vnode), pMeterMeta->sid, pMeterMetaInfo->name, pMeterMeta->numOfColumns, pCol->colIndex, + htons(pQueryMsg->vnode), pTableMeta->sid, pTableMetaInfo->name, tscGetNumOfColumns(pTableMeta), pCol->colIndex, pColSchema->name); return -1; // 0 means build msg failed @@ -1537,7 +817,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { hasArithmeticFunction = true; } - if (!tscValidateColumnId(pMeterMetaInfo, pExpr->colInfo.colId)) { + if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId)) { /* column id is not valid according to the cached metermeta, the meter meta is expired */ tscError("%p table schema is not matched with parsed sql", pSql); return -1; @@ -1588,17 +868,17 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pMsg = doSerializeTableInfo(pSql, numOfTables, htons(pQueryMsg->vnode), pMsg); // only include the required tag column schema. If a tag is not required, it won't be sent to vnode - if (pMeterMetaInfo->numOfTags > 0) { + if (pTableMetaInfo->numOfTags > 0) { // always transfer tag schema to vnode if exists - SSchema *pTagSchema = tsGetTagSchema(pMeterMeta); + SSchema *pTagSchema = tscGetTableTagSchema(pTableMeta); - for (int32_t j = 0; j < pMeterMetaInfo->numOfTags; ++j) { - if (pMeterMetaInfo->tagColumnIndex[j] == TSDB_TBNAME_COLUMN_INDEX) { + for (int32_t j = 0; j < pTableMetaInfo->numOfTags; ++j) { + if (pTableMetaInfo->tagColumnIndex[j] == TSDB_TBNAME_COLUMN_INDEX) { SSchema tbSchema = { .bytes = TSDB_TABLE_NAME_LEN, .colId = TSDB_TBNAME_COLUMN_INDEX, .type = TSDB_DATA_TYPE_BINARY}; memcpy(pMsg, &tbSchema, sizeof(SSchema)); } else { - memcpy(pMsg, &pTagSchema[pMeterMetaInfo->tagColumnIndex[j]], sizeof(SSchema)); + memcpy(pMsg, &pTagSchema[pTableMetaInfo->tagColumnIndex[j]], sizeof(SSchema)); } pMsg += sizeof(SSchema); @@ -1643,7 +923,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t numOfBlocks = 0; if (pQueryInfo->tsBuf != NULL) { - STSVnodeBlockInfo *pBlockInfo = tsBufGetVnodeBlockInfo(pQueryInfo->tsBuf, pMeterMetaInfo->vnodeIndex); + STSVnodeBlockInfo *pBlockInfo = tsBufGetVnodeBlockInfo(pQueryInfo->tsBuf, pTableMetaInfo->vnodeIndex); assert(QUERY_IS_JOIN_QUERY(pQueryInfo->type) && pBlockInfo != NULL); // this query should not be sent // todo refactor @@ -1685,8 +965,8 @@ int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SCMCreateDbMsg *pCreateDbMsg = (SCMCreateDbMsg*)pCmd->payload; assert(pCmd->numOfClause == 1); - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); - strncpy(pCreateDbMsg->db, pMeterMetaInfo->name, tListLen(pCreateDbMsg->db)); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + strncpy(pCreateDbMsg->db, pTableMetaInfo->name, tListLen(pCreateDbMsg->db)); return TSDB_CODE_SUCCESS; } @@ -1807,8 +1087,8 @@ int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SCMDropDbMsg *pDropDbMsg = (SCMDropDbMsg*)pCmd->payload; - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); - strncpy(pDropDbMsg->db, pMeterMetaInfo->name, tListLen(pDropDbMsg->db)); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + strncpy(pDropDbMsg->db, pTableMetaInfo->name, tListLen(pDropDbMsg->db)); pDropDbMsg->ignoreNotExists = pInfo->pDCLInfo->existsCheck ? 1 : 0; pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_DB; @@ -1825,8 +1105,8 @@ int32_t tscBuildDropTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } SCMDropTableMsg *pDropTableMsg = (SCMDropTableMsg*)pCmd->payload; - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); - strcpy(pDropTableMsg->tableId, pMeterMetaInfo->name); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + strcpy(pDropTableMsg->tableId, pTableMetaInfo->name); pDropTableMsg->igNotExists = pInfo->pDCLInfo->existsCheck ? 1 : 0; pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_TABLE; @@ -1842,8 +1122,8 @@ int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } SCMDropDnodeMsg *pDrop = (SCMDropDnodeMsg *)pCmd->payload; - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); - strcpy(pDrop->ip, pMeterMetaInfo->name); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + strcpy(pDrop->ip, pTableMetaInfo->name); pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_DNODE; return TSDB_CODE_SUCCESS; @@ -1860,8 +1140,8 @@ int32_t tscBuildDropAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } SCMDropUserMsg *pDropMsg = (SCMDropUserMsg*)pCmd->payload; - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); - strcpy(pDropMsg->user, pMeterMetaInfo->name); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + strcpy(pDropMsg->user, pTableMetaInfo->name); return TSDB_CODE_SUCCESS; } @@ -1876,8 +1156,8 @@ int32_t tscBuildUseDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } SCMUseDbMsg *pUseDbMsg = (SCMUseDbMsg*)pCmd->payload; - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); - strcpy(pUseDbMsg->db, pMeterMetaInfo->name); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + strcpy(pUseDbMsg->db, pTableMetaInfo->name); pCmd->msgType = TSDB_MSG_TYPE_CM_USE_DB; return TSDB_CODE_SUCCESS; @@ -1896,10 +1176,10 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SCMShowMsg *pShowMsg = (SCMShowMsg*)pCmd->payload; - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); - size_t nameLen = strlen(pMeterMetaInfo->name); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + size_t nameLen = strlen(pTableMetaInfo->name); if (nameLen > 0) { - strcpy(pShowMsg->db, pMeterMetaInfo->name); // prefix is set here + strcpy(pShowMsg->db, pTableMetaInfo->name); // prefix is set here } else { strcpy(pShowMsg->db, pObj->db); } @@ -1976,7 +1256,7 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); // Reallocate the payload size size = tscEstimateCreateTableMsgLength(pSql, pInfo); @@ -1987,10 +1267,10 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SCMCreateTableMsg *pCreateTableMsg = (SCMCreateTableMsg *)pCmd->payload; - strcpy(pCreateTableMsg->tableId, pMeterMetaInfo->name); + strcpy(pCreateTableMsg->tableId, pTableMetaInfo->name); // use dbinfo from table id without modifying current db info - tscGetDBInfoFromMeterId(pMeterMetaInfo->name, pCreateTableMsg->db); + tscGetDBInfoFromMeterId(pTableMetaInfo->name, pCreateTableMsg->db); SCreateTableSQL *pCreateTable = pInfo->pCreateTableInfo; @@ -2055,7 +1335,7 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd * pCmd = &pSql->cmd; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); size = tscEstimateAlterTableMsgLength(pCmd); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { @@ -2065,11 +1345,11 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pAlterTableMsg = (SCMAlterTableMsg *)pCmd->payload; - tscGetDBInfoFromMeterId(pMeterMetaInfo->name, pAlterTableMsg->db); + tscGetDBInfoFromMeterId(pTableMetaInfo->name, pAlterTableMsg->db); SAlterTableSQL *pAlterInfo = pInfo->pAlterInfo; - strcpy(pAlterTableMsg->tableId, pMeterMetaInfo->name); + strcpy(pAlterTableMsg->tableId, pTableMetaInfo->name); pAlterTableMsg->type = htons(pAlterInfo->type); pAlterTableMsg->numOfCols = htons(tscNumOfFields(pQueryInfo)); @@ -2107,8 +1387,8 @@ int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } SCMAlterDbMsg *pAlterDbMsg = (SCMAlterDbMsg*)pCmd->payload; - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); - strcpy(pAlterDbMsg->db, pMeterMetaInfo->name); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + strcpy(pAlterDbMsg->db, pTableMetaInfo->name); return TSDB_CODE_SUCCESS; } @@ -2182,10 +1462,11 @@ static int tscLocalResultCommonBuilder(SSqlObj *pSql, int32_t numOfRes) { int tscProcessDescribeTableRsp(SSqlObj *pSql) { SSqlCmd * pCmd = &pSql->cmd; - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); - - int32_t numOfRes = pMeterMetaInfo->pMeterMeta->numOfColumns + pMeterMetaInfo->pMeterMeta->numOfTags; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); + + int32_t numOfRes = tinfo.numOfColumns + tinfo.numOfTags; return tscLocalResultCommonBuilder(pSql, numOfRes); } @@ -2193,11 +1474,11 @@ int tscProcessTagRetrieveRsp(SSqlObj *pSql) { SSqlCmd *pCmd = &pSql->cmd; SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); int32_t numOfRes = 0; if (tscSqlExprGet(pQueryInfo, 0)->functionId == TSDB_FUNC_TAGPRJ) { - numOfRes = pMeterMetaInfo->pMetricMeta->numOfTables; + numOfRes = pTableMetaInfo->pMetricMeta->numOfTables; } else { numOfRes = 1; // for count function, there is only one output. } @@ -2254,7 +1535,7 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) { return TSDB_CODE_SUCCESS; } -int tscBuildMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { +int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SCMTableInfoMsg *pInfoMsg; char * pMsg; int msgLen = 0; @@ -2273,15 +1554,15 @@ int tscBuildMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd * pCmd = &pSql->cmd; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); pInfoMsg = (SCMTableInfoMsg *)pCmd->payload; - strcpy(pInfoMsg->tableId, pMeterMetaInfo->name); - pInfoMsg->createFlag = htons(pSql->cmd.createOnDemand ? 1 : 0); + strcpy(pInfoMsg->tableId, pTableMetaInfo->name); + pInfoMsg->createFlag = htons(pSql->cmd.autoCreated ? 1 : 0); pMsg = (char*)pInfoMsg + sizeof(SCMTableInfoMsg); - if (pSql->cmd.createOnDemand) { + if (pSql->cmd.autoCreated) { memcpy(pInfoMsg->tags, tmpData, sizeof(STagData)); pMsg += sizeof(STagData); } @@ -2372,7 +1653,7 @@ int tscBuildMetricMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { STagCond *pTagCond = &pQueryInfo->tagCond; - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, tableIndex); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); int32_t size = tscEstimateMetricMetaMsgSize(pCmd); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { @@ -2384,7 +1665,7 @@ int tscBuildMetricMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pMsg = pStart; SMgmtHead *pMgmt = (SMgmtHead *)pMsg; - tscGetDBInfoFromMeterId(pMeterMetaInfo->name, pMgmt->db); + tscGetDBInfoFromMeterId(pTableMetaInfo->name, pMgmt->db); pMsg += sizeof(SMgmtHead); @@ -2412,8 +1693,8 @@ int tscBuildMetricMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pMsg += sizeof(int16_t); for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, i); - uint64_t uid = pMeterMetaInfo->pMeterMeta->uid; + pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, i); + uint64_t uid = pTableMetaInfo->pTableMeta->uid; offset = pMsg - (char *)pMetaMsg; pMetaMsg->metaElem[i] = htonl(offset); @@ -2466,8 +1747,8 @@ int tscBuildMetricMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pElem->numOfGroupCols = 0; } else { pElem->numOfGroupCols = htons(pGroupby->numOfGroupCols); - for (int32_t j = 0; j < pMeterMetaInfo->numOfTags; ++j) { - pElem->tagCols[j] = htons(pMeterMetaInfo->tagColumnIndex[j]); + for (int32_t j = 0; j < pTableMetaInfo->numOfTags; ++j) { + pElem->tagCols[j] = htons(pTableMetaInfo->tagColumnIndex[j]); } if (pGroupby->numOfGroupCols != 0) { @@ -2491,8 +1772,8 @@ int tscBuildMetricMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } } - strcpy(pElem->tableId, pMeterMetaInfo->name); - pElem->numOfTags = htons(pMeterMetaInfo->numOfTags); + strcpy(pElem->tableId, pTableMetaInfo->name); + pElem->numOfTags = htons(pTableMetaInfo->numOfTags); int16_t len = pMsg - (char *)pElem; pElem->elemLen = htons(len); // redundant data for integrate check @@ -2563,81 +1844,76 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { return msgLen; } -int tscProcessMeterMetaRsp(SSqlObj *pSql) { - STableMeta *pMeta; - SSchema * pSchema; - - pMeta = (STableMeta *)pSql->res.pRsp; +int tscProcessTableMetaRsp(SSqlObj *pSql) { + STableMetaMsg *pMetaMsg = (STableMetaMsg *)pSql->res.pRsp; - pMeta->sid = htonl(pMeta->sid); - pMeta->sversion = htons(pMeta->sversion); - pMeta->vgid = htonl(pMeta->vgid); - pMeta->uid = htobe64(pMeta->uid); - pMeta->contLen = htons(pMeta->contLen); + pMetaMsg->sid = htonl(pMetaMsg->sid); + pMetaMsg->sversion = htons(pMetaMsg->sversion); + pMetaMsg->vgid = htonl(pMetaMsg->vgid); + pMetaMsg->uid = htobe64(pMetaMsg->uid); + pMetaMsg->contLen = htons(pMetaMsg->contLen); - if (pMeta->sid < 0 || pMeta->vgid < 0) { - tscError("invalid meter vgid:%d, sid%d", pMeta->vgid, pMeta->sid); + if (pMetaMsg->sid < 0 || pMetaMsg->vgid < 0) { + tscError("invalid meter vgid:%d, sid%d", pMetaMsg->vgid, pMetaMsg->sid); return TSDB_CODE_INVALID_VALUE; } - pMeta->numOfColumns = htons(pMeta->numOfColumns); + pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); - if (pMeta->numOfTags > TSDB_MAX_TAGS || pMeta->numOfTags < 0) { - tscError("invalid numOfTags:%d", pMeta->numOfTags); + if (pMetaMsg->numOfTags > TSDB_MAX_TAGS || pMetaMsg->numOfTags < 0) { + tscError("invalid numOfTags:%d", pMetaMsg->numOfTags); return TSDB_CODE_INVALID_VALUE; } - if (pMeta->numOfColumns > TSDB_MAX_COLUMNS || pMeta->numOfColumns <= 0) { - tscError("invalid numOfColumns:%d", pMeta->numOfColumns); + if (pMetaMsg->numOfColumns > TSDB_MAX_COLUMNS || pMetaMsg->numOfColumns <= 0) { + tscError("invalid numOfColumns:%d", pMetaMsg->numOfColumns); return TSDB_CODE_INVALID_VALUE; } for (int i = 0; i < TSDB_VNODES_SUPPORT; ++i) { - pMeta->vpeerDesc[i].vnode = htonl(pMeta->vpeerDesc[i].vnode); + pMetaMsg->vpeerDesc[i].vnode = htonl(pMetaMsg->vpeerDesc[i].vnode); } - pMeta->rowSize = 0; - pSchema = (SSchema *)(pSql->res.pRsp + sizeof(STableMeta)); + SSchema* pSchema = pMetaMsg->schema; - int32_t numOfTotalCols = pMeta->numOfColumns + pMeta->numOfTags; + int32_t numOfTotalCols = pMetaMsg->numOfColumns + pMetaMsg->numOfTags; for (int i = 0; i < numOfTotalCols; ++i) { pSchema->bytes = htons(pSchema->bytes); pSchema->colId = htons(pSchema->colId); - - // ignore the tags length - if (i < pMeta->numOfColumns) { - pMeta->rowSize += pSchema->bytes; - } pSchema++; } // rsp += numOfTotalCols * sizeof(SSchema); // // int32_t tagLen = 0; -// SSchema *pTagsSchema = tsGetTagSchema(pMeta); +// SSchema *pTagsSchema = tscGetTableTagSchema(pMetaMsg); // -// if (pMeta->tableType == TSDB_CHILD_TABLE) { -// for (int32_t i = 0; i < pMeta->numOfTags; ++i) { +// if (pMetaMsg->tableType == TSDB_CHILD_TABLE) { +// for (int32_t i = 0; i < pMetaMsg->numOfTags; ++i) { // tagLen += pTagsSchema[i].bytes; // } // } // // rsp += tagLen; -// int32_t size = (int32_t)(rsp - (char *)pMeta); +// int32_t size = (int32_t)(rsp - (char *)pMetaMsg); - // pMeta->index = rand() % TSDB_VNODES_SUPPORT; -// pMeta->index = 0; + size_t size = 0; + STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size); // todo add one more function: taosAddDataIfNotExists(); - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, 0, 0); - assert(pMeterMetaInfo->pMeterMeta == NULL); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); + assert(pTableMetaInfo->pTableMeta == NULL); - pMeterMetaInfo->pMeterMeta = (STableMeta *)taosCachePut(tscCacheHandle, pMeterMetaInfo->name, (char *)pMeta, - pMeta->contLen, tsMeterMetaKeepTimer); + pTableMetaInfo->pTableMeta = + (STableMeta *) taosCachePut(tscCacheHandle, pTableMetaInfo->name, pTableMeta, size, tsMeterMetaKeepTimer); + // todo handle out of memory case - if (pMeterMetaInfo->pMeterMeta == NULL) return 0; + if (pTableMetaInfo->pTableMeta == NULL) { + return TSDB_CODE_CLI_OUT_OF_MEMORY; + } - return TSDB_CODE_OTHERS; + free(pTableMeta); + return TSDB_CODE_SUCCESS; } /** @@ -2646,7 +1922,6 @@ int tscProcessMeterMetaRsp(SSqlObj *pSql) { * |...... 1B 1B 4B **/ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) { - SSchema *pSchema; uint8_t ieType; int32_t totalNum; int32_t i; @@ -2683,67 +1958,68 @@ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) { return TSDB_CODE_OTHERS; } - pMeta->numOfColumns = htons(pMeta->numOfColumns); - - if (pMeta->numOfTags > TSDB_MAX_TAGS || pMeta->numOfTags < 0) { - tscError("invalid tag value count:%d", pMeta->numOfTags); - pSql->res.code = TSDB_CODE_INVALID_VALUE; - pSql->res.numOfTotal = i; - return TSDB_CODE_OTHERS; - } - - if (pMeta->numOfTags > TSDB_MAX_TAGS || pMeta->numOfTags < 0) { - tscError("invalid numOfTags:%d", pMeta->numOfTags); - pSql->res.code = TSDB_CODE_INVALID_VALUE; - pSql->res.numOfTotal = i; - return TSDB_CODE_OTHERS; - } - - if (pMeta->numOfColumns > TSDB_MAX_COLUMNS || pMeta->numOfColumns < 0) { - tscError("invalid numOfColumns:%d", pMeta->numOfColumns); - pSql->res.code = TSDB_CODE_INVALID_VALUE; - pSql->res.numOfTotal = i; - return TSDB_CODE_OTHERS; - } - - for (int j = 0; j < TSDB_VNODES_SUPPORT; ++j) { - pMeta->vpeerDesc[j].vnode = htonl(pMeta->vpeerDesc[j].vnode); - } - - pMeta->rowSize = 0; - rsp += sizeof(SMultiTableMeta); - pSchema = (SSchema *)rsp; - - int32_t numOfTotalCols = pMeta->numOfColumns + pMeta->numOfTags; - for (int j = 0; j < numOfTotalCols; ++j) { - pSchema->bytes = htons(pSchema->bytes); - pSchema->colId = htons(pSchema->colId); - - // ignore the tags length - if (j < pMeta->numOfColumns) { - pMeta->rowSize += pSchema->bytes; - } - pSchema++; - } - - rsp += numOfTotalCols * sizeof(SSchema); - - int32_t tagLen = 0; - SSchema *pTagsSchema = tsGetTagSchema(pMeta); - - if (pMeta->tableType == TSDB_CHILD_TABLE) { - for (int32_t j = 0; j < pMeta->numOfTags; ++j) { - tagLen += pTagsSchema[j].bytes; - } - } - - rsp += tagLen; - int32_t size = (int32_t)(rsp - ((char *)pMeta)); // Consistent with STableMeta in cache - - pMeta->index = 0; - (void)taosCachePut(tscCacheHandle, pMeta->tableId, (char *)pMeta, size, tsMeterMetaKeepTimer); + // pMeta->numOfColumns = htons(pMeta->numOfColumns); + // + // if (pMeta->numOfTags > TSDB_MAX_TAGS || pMeta->numOfTags < 0) { + // tscError("invalid tag value count:%d", pMeta->numOfTags); + // pSql->res.code = TSDB_CODE_INVALID_VALUE; + // pSql->res.numOfTotal = i; + // return TSDB_CODE_OTHERS; + // } + // + // if (pMeta->numOfTags > TSDB_MAX_TAGS || pMeta->numOfTags < 0) { + // tscError("invalid numOfTags:%d", pMeta->numOfTags); + // pSql->res.code = TSDB_CODE_INVALID_VALUE; + // pSql->res.numOfTotal = i; + // return TSDB_CODE_OTHERS; + // } + // + // if (pMeta->numOfColumns > TSDB_MAX_COLUMNS || pMeta->numOfColumns < 0) { + // tscError("invalid numOfColumns:%d", pMeta->numOfColumns); + // pSql->res.code = TSDB_CODE_INVALID_VALUE; + // pSql->res.numOfTotal = i; + // return TSDB_CODE_OTHERS; + // } + // + // for (int j = 0; j < TSDB_VNODES_SUPPORT; ++j) { + // pMeta->vpeerDesc[j].vnode = htonl(pMeta->vpeerDesc[j].vnode); + // } + // + // pMeta->rowSize = 0; + // rsp += sizeof(SMultiTableMeta); + // pSchema = (SSchema *)rsp; + // + // int32_t numOfTotalCols = pMeta->numOfColumns + pMeta->numOfTags; + // for (int j = 0; j < numOfTotalCols; ++j) { + // pSchema->bytes = htons(pSchema->bytes); + // pSchema->colId = htons(pSchema->colId); + // + // // ignore the tags length + // if (j < pMeta->numOfColumns) { + // pMeta->rowSize += pSchema->bytes; + // } + // pSchema++; + // } + // + // rsp += numOfTotalCols * sizeof(SSchema); + // + // int32_t tagLen = 0; + // SSchema *pTagsSchema = tscGetTableTagSchema(pMeta); + // + // if (pMeta->tableType == TSDB_CHILD_TABLE) { + // for (int32_t j = 0; j < pMeta->numOfTags; ++j) { + // tagLen += pTagsSchema[j].bytes; + // } + // } + // + // rsp += tagLen; + // int32_t size = (int32_t)(rsp - ((char *)pMeta)); // Consistent with STableMeta in cache + // + // pMeta->index = 0; + // (void)taosCachePut(tscCacheHandle, pMeta->tableId, (char *)pMeta, size, tsMeterMetaKeepTimer); + // } } - + pSql->res.code = TSDB_CODE_SUCCESS; pSql->res.numOfTotal = i; tscTrace("%p load multi-metermeta resp complete num:%d", pSql, pSql->res.numOfTotal); @@ -2840,22 +2116,22 @@ int tscProcessMetricMetaRsp(SSqlObj *pSql) { for (int32_t i = 0; i < num; ++i) { char name[TSDB_MAX_TAGS_LEN + 1] = {0}; - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, i); - tscGetMetricMetaCacheKey(pQueryInfo, name, pMeterMetaInfo->pMeterMeta->uid); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); + tscGetMetricMetaCacheKey(pQueryInfo, name, pTableMetaInfo->pTableMeta->uid); #ifdef _DEBUG_VIEW printf("generate the metric key:%s, index:%d\n", name, i); #endif // release the used metricmeta - taosCacheRelease(tscCacheHandle, (void **)&(pMeterMetaInfo->pMetricMeta), false); + taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), false); - pMeterMetaInfo->pMetricMeta = (SSuperTableMeta *)taosCachePut(tscCacheHandle, name, (char *)metricMetaList[i], + pTableMetaInfo->pMetricMeta = (SSuperTableMeta *)taosCachePut(tscCacheHandle, name, (char *)metricMetaList[i], sizes[i], tsMetricMetaKeepTimer); tfree(metricMetaList[i]); // failed to put into cache - if (pMeterMetaInfo->pMetricMeta == NULL) { + if (pTableMetaInfo->pMetricMeta == NULL) { pSql->res.code = TSDB_CODE_CLI_OUT_OF_MEMORY; goto _error_clean; } @@ -2877,7 +2153,7 @@ _error_clean: * current process do not use the cache at all */ int tscProcessShowRsp(SSqlObj *pSql) { - STableMeta * pMeta; + STableMetaMsg * pMetaMsg; SCMShowRsp *pShow; SSchema * pSchema; char key[20]; @@ -2885,22 +2161,22 @@ int tscProcessShowRsp(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); //? + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); pShow = (SCMShowRsp *)pRes->pRsp; pShow->qhandle = htobe64(pShow->qhandle); pRes->qhandle = pShow->qhandle; tscResetForNextRetrieve(pRes); - pMeta = &(pShow->tableMeta); + pMetaMsg = &(pShow->tableMeta); - pMeta->numOfColumns = ntohs(pMeta->numOfColumns); + pMetaMsg->numOfColumns = ntohs(pMetaMsg->numOfColumns); - pSchema = (SSchema *)((char *)pMeta + sizeof(STableMeta)); - pMeta->sid = ntohs(pMeta->sid); - for (int i = 0; i < pMeta->numOfColumns; ++i) { + pSchema = pMetaMsg->schema; + pMetaMsg->sid = ntohs(pMetaMsg->sid); + for (int i = 0; i < pMetaMsg->numOfColumns; ++i) { pSchema->bytes = htons(pSchema->bytes); pSchema++; } @@ -2908,27 +2184,31 @@ int tscProcessShowRsp(SSqlObj *pSql) { key[0] = pCmd->msgType + 'a'; strcpy(key + 1, "showlist"); - taosCacheRelease(tscCacheHandle, (void *)&(pMeterMetaInfo->pMeterMeta), false); - - int32_t size = pMeta->numOfColumns * sizeof(SSchema) + sizeof(STableMeta); - pMeterMetaInfo->pMeterMeta = - (STableMeta *)taosCachePut(tscCacheHandle, key, (char *)pMeta, size, tsMeterMetaKeepTimer); + taosCacheRelease(tscCacheHandle, (void *)&(pTableMetaInfo->pTableMeta), false); + size_t size = 0; + STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size); + + pTableMetaInfo->pTableMeta = + (STableMeta *)taosCachePut(tscCacheHandle, key, (char *)pTableMeta, size, tsMeterMetaKeepTimer); + pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutputCols; - SSchema *pMeterSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); + SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - tscColumnBaseInfoReserve(&pQueryInfo->colList, pMeta->numOfColumns); + tscColumnBaseInfoReserve(&pQueryInfo->colList, pMetaMsg->numOfColumns); SColumnIndex index = {0}; - for (int16_t i = 0; i < pMeta->numOfColumns; ++i) { + for (int16_t i = 0; i < pMetaMsg->numOfColumns; ++i) { index.columnIndex = i; tscColumnBaseInfoInsert(pQueryInfo, &index); - tscFieldInfoSetValFromSchema(&pQueryInfo->fieldsInfo, i, &pMeterSchema[i]); + tscFieldInfoSetValFromSchema(&pQueryInfo->fieldsInfo, i, &pTableSchema[i]); pQueryInfo->fieldsInfo.pSqlExpr[i] = tscSqlExprInsert(pQueryInfo, i, TSDB_FUNC_TS_DUMMY, &index, - pMeterSchema[i].type, pMeterSchema[i].bytes, pMeterSchema[i].bytes); + pTableSchema[i].type, pTableSchema[i].bytes, pTableSchema[i].bytes); } tscFieldInfoCalOffset(pQueryInfo); + + tfree(pTableMeta); return 0; } @@ -2959,9 +2239,9 @@ int tscProcessConnectRsp(SSqlObj *pSql) { int tscProcessUseDbRsp(SSqlObj *pSql) { STscObj * pObj = pSql->pTscObj; - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, 0, 0); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - strcpy(pObj->db, pMeterMetaInfo->name); + strcpy(pObj->db, pTableMetaInfo->name); return 0; } @@ -2971,10 +2251,10 @@ int tscProcessDropDbRsp(SSqlObj *UNUSED_PARAM(pSql)) { } int tscProcessDropTableRsp(SSqlObj *pSql) { - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, 0, 0); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - STableMeta *pMeterMeta = taosCacheAcquireByName(tscCacheHandle, pMeterMetaInfo->name); - if (pMeterMeta == NULL) { + STableMeta *pTableMeta = taosCacheAcquireByName(tscCacheHandle, pTableMetaInfo->name); + if (pTableMeta == NULL) { /* not in cache, abort */ return 0; } @@ -2986,36 +2266,36 @@ int tscProcessDropTableRsp(SSqlObj *pSql) { * The cached information is expired, however, we may have lost the ref of original meter. So, clear whole cache * instead. */ - tscTrace("%p force release metermeta after drop table:%s", pSql, pMeterMetaInfo->name); - taosCacheRelease(tscCacheHandle, (void **)&pMeterMeta, true); + tscTrace("%p force release metermeta after drop table:%s", pSql, pTableMetaInfo->name); + taosCacheRelease(tscCacheHandle, (void **)&pTableMeta, true); - if (pMeterMetaInfo->pMeterMeta) { - taosCacheRelease(tscCacheHandle, (void **)&(pMeterMetaInfo->pMeterMeta), true); - taosCacheRelease(tscCacheHandle, (void **)&(pMeterMetaInfo->pMetricMeta), true); + if (pTableMetaInfo->pTableMeta) { + taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true); + taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), true); } return 0; } int tscProcessAlterTableMsgRsp(SSqlObj *pSql) { - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, 0, 0); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - STableMeta *pMeterMeta = taosCacheAcquireByName(tscCacheHandle, pMeterMetaInfo->name); - if (pMeterMeta == NULL) { /* not in cache, abort */ + STableMeta *pTableMeta = taosCacheAcquireByName(tscCacheHandle, pTableMetaInfo->name); + if (pTableMeta == NULL) { /* not in cache, abort */ return 0; } - tscTrace("%p force release metermeta in cache after alter-table: %s", pSql, pMeterMetaInfo->name); - taosCacheRelease(tscCacheHandle, (void **)&pMeterMeta, true); + tscTrace("%p force release metermeta in cache after alter-table: %s", pSql, pTableMetaInfo->name); + taosCacheRelease(tscCacheHandle, (void **)&pTableMeta, true); - if (pMeterMetaInfo->pMeterMeta) { - bool isSuperTable = UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo); + if (pTableMetaInfo->pTableMeta) { + bool isSuperTable = UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo); - taosCacheRelease(tscCacheHandle, (void **)&(pMeterMetaInfo->pMeterMeta), true); - taosCacheRelease(tscCacheHandle, (void **)&(pMeterMetaInfo->pMetricMeta), true); + taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true); + taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), true); if (isSuperTable) { // if it is a super table, reset whole query cache - tscTrace("%p reset query cache since table:%s is stable", pSql, pMeterMetaInfo->name); + tscTrace("%p reset query cache since table:%s is stable", pSql, pTableMetaInfo->name); taosCacheEmpty(tscCacheHandle); } } @@ -3096,11 +2376,9 @@ int tscProcessRetrieveRspFromLocal(SSqlObj *pSql) { return 0; } -void tscMeterMetaCallBack(void *param, TAOS_RES *res, int code); - -static int32_t doGetMeterMetaFromServer(SSqlObj *pSql, SMeterMetaInfo *pMeterMetaInfo) { - int32_t code = TSDB_CODE_SUCCESS; +void tscTableMetaCallBack(void *param, TAOS_RES *res, int code); +static int32_t doGetMeterMetaFromServer(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); if (NULL == pNew) { tscError("%p malloc failed for new sqlobj to get meter meta", pSql); @@ -3116,7 +2394,7 @@ static int32_t doGetMeterMetaFromServer(SSqlObj *pSql, SMeterMetaInfo *pMeterMet SQueryInfo *pNewQueryInfo = NULL; tscGetQueryInfoDetailSafely(&pNew->cmd, 0, &pNewQueryInfo); - pNew->cmd.createOnDemand = pSql->cmd.createOnDemand; // create table if not exists + pNew->cmd.autoCreated = pSql->cmd.autoCreated; // create table if not exists if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { tscError("%p malloc failed for payload to get meter meta", pSql); free(pNew); @@ -3124,73 +2402,51 @@ static int32_t doGetMeterMetaFromServer(SSqlObj *pSql, SMeterMetaInfo *pMeterMet return TSDB_CODE_CLI_OUT_OF_MEMORY; } - SMeterMetaInfo *pNewMeterMetaInfo = tscAddEmptyMeterMetaInfo(pNewQueryInfo); + STableMetaInfo *pNewMeterMetaInfo = tscAddEmptyMetaInfo(pNewQueryInfo); assert(pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1); - strcpy(pNewMeterMetaInfo->name, pMeterMetaInfo->name); + strcpy(pNewMeterMetaInfo->name, pTableMetaInfo->name); memcpy(pNew->cmd.payload, pSql->cmd.payload, TSDB_DEFAULT_PAYLOAD_SIZE); // tag information if table does not exists. tscTrace("%p new pSqlObj:%p to get tableMeta", pSql, pNew); - if (pSql->fp == NULL) { - tsem_init(&pNew->rspSem, 0, 0); - tsem_init(&pNew->emptyRspSem, 0, 1); - - code = tscProcessSql(pNew); - - /* - * Update cache only on succeeding in getting metermeta. - * Transfer the ownership of metermeta to the new object, instead of invoking the release/acquire routine - */ - if (code == TSDB_CODE_SUCCESS) { - pMeterMetaInfo->pMeterMeta = taosCacheTransfer(tscCacheHandle, (void**) &pNewMeterMetaInfo->pMeterMeta); - assert(pMeterMetaInfo->pMeterMeta != NULL); - } - - tscTrace("%p get meter meta complete, code:%d, pMeterMeta:%p", pSql, code, pMeterMetaInfo->pMeterMeta); - tscFreeSqlObj(pNew); - - } else { - pNew->fp = tscMeterMetaCallBack; - pNew->param = pSql; - pNew->sqlstr = strdup(pSql->sqlstr); + pNew->fp = tscTableMetaCallBack; + pNew->param = pSql; - code = tscProcessSql(pNew); - if (code == TSDB_CODE_SUCCESS) { - code = TSDB_CODE_ACTION_IN_PROGRESS; - } + int32_t code = tscProcessSql(pNew); + if (code == TSDB_CODE_SUCCESS) { + code = TSDB_CODE_ACTION_IN_PROGRESS; } return code; } -int tscGetMeterMeta(SSqlObj *pSql, SMeterMetaInfo *pMeterMetaInfo) { - assert(strlen(pMeterMetaInfo->name) != 0); +int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { + assert(strlen(pTableMetaInfo->name) != 0); - // If this SMeterMetaInfo owns a metermeta, release it first - if (pMeterMetaInfo->pMeterMeta != NULL) { - taosCacheRelease(tscCacheHandle, (void **)&(pMeterMetaInfo->pMeterMeta), false); + // If this STableMetaInfo owns a metermeta, release it first + if (pTableMetaInfo->pTableMeta != NULL) { + taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), false); } - pMeterMetaInfo->pMeterMeta = (STableMeta *)taosCacheAcquireByName(tscCacheHandle, pMeterMetaInfo->name); - if (pMeterMetaInfo->pMeterMeta != NULL) { - STableMeta *pMeterMeta = pMeterMetaInfo->pMeterMeta; - - tscTrace("%p retrieve tableMeta from cache, the number of columns:%d, numOfTags:%d", pSql, pMeterMeta->numOfColumns, - pMeterMeta->numOfTags); + pTableMetaInfo->pTableMeta = (STableMeta *)taosCacheAcquireByName(tscCacheHandle, pTableMetaInfo->name); + if (pTableMetaInfo->pTableMeta != NULL) { + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); + tscTrace("%p retrieve tableMeta from cache, the number of columns:%d, numOfTags:%d", pSql, tinfo.numOfColumns, + tinfo.numOfTags); return TSDB_CODE_SUCCESS; } /* * for async insert operation, release data block buffer before issue new object to get metermeta - * because in metermeta callback function, the tscParse function will generate the submit data blocks + * because in table meta callback function, the tscParse function will generate the submit data blocks */ - return doGetMeterMetaFromServer(pSql, pMeterMetaInfo); + return doGetMeterMetaFromServer(pSql, pTableMetaInfo); } -int tscGetMeterMetaEx(SSqlObj *pSql, SMeterMetaInfo *pMeterMetaInfo, bool createIfNotExists) { - pSql->cmd.createOnDemand = createIfNotExists; - return tscGetMeterMeta(pSql, pMeterMetaInfo); +int tscGetMeterMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool createIfNotExists) { + pSql->cmd.autoCreated = createIfNotExists; + return tscGetTableMeta(pSql, pTableMetaInfo); } /* @@ -3219,35 +2475,26 @@ int tscRenewMeterMeta(SSqlObj *pSql, char *tableId) { SSqlCmd *pCmd = &pSql->cmd; SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - - // enforce the renew metermeta operation in async model - if (pSql->fp == NULL) pSql->fp = (void *)0x1; + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); /* * 1. only update the metermeta in force model metricmeta is not updated * 2. if get metermeta failed, still get the metermeta */ - if (pMeterMetaInfo->pMeterMeta == NULL || !tscQueryOnMetric(pCmd)) { - if (pMeterMetaInfo->pMeterMeta) { + if (pTableMetaInfo->pTableMeta == NULL || !tscQueryOnMetric(pCmd)) { + if (pTableMetaInfo->pTableMeta) { tscTrace("%p update meter meta, old: numOfTags:%d, numOfCols:%d, uid:%" PRId64 ", addr:%p", pSql, - pMeterMetaInfo->numOfTags, pCmd->numOfCols, pMeterMetaInfo->pMeterMeta->uid, pMeterMetaInfo->pMeterMeta); + pTableMetaInfo->numOfTags, pCmd->numOfCols, pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->pTableMeta); } tscWaitingForCreateTable(pCmd); - taosCacheRelease(tscCacheHandle, (void **)&(pMeterMetaInfo->pMeterMeta), true); + taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true); - code = doGetMeterMetaFromServer(pSql, pMeterMetaInfo); // todo ?? + code = doGetMeterMetaFromServer(pSql, pTableMetaInfo); // todo ?? } else { tscTrace("%p metric query not update metric meta, numOfTags:%d, numOfCols:%d, uid:%" PRId64 ", addr:%p", pSql, - pMeterMetaInfo->pMeterMeta->numOfTags, pCmd->numOfCols, pMeterMetaInfo->pMeterMeta->uid, - pMeterMetaInfo->pMeterMeta); - } - - if (code != TSDB_CODE_ACTION_IN_PROGRESS) { - if (pSql->fp == (void *)0x1) { - pSql->fp = NULL; - } + tscGetNumOfTags(pTableMetaInfo->pTableMeta), pCmd->numOfCols, pTableMetaInfo->pTableMeta->uid, + pTableMetaInfo->pTableMeta); } return code; @@ -3266,17 +2513,17 @@ int tscGetMetricMeta(SSqlObj *pSql, int32_t clauseIndex) { for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { char tagstr[TSDB_MAX_TAGS_LEN + 1] = {0}; - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, i); - tscGetMetricMetaCacheKey(pQueryInfo, tagstr, pMeterMetaInfo->pMeterMeta->uid); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); + tscGetMetricMetaCacheKey(pQueryInfo, tagstr, pTableMetaInfo->pTableMeta->uid); - taosCacheRelease(tscCacheHandle, (void **)&(pMeterMetaInfo->pMetricMeta), false); + taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), false); SSuperTableMeta *ppMeta = (SSuperTableMeta *)taosCacheAcquireByName(tscCacheHandle, tagstr); if (ppMeta == NULL) { required = true; break; } else { - pMeterMetaInfo->pMetricMeta = ppMeta; + pTableMetaInfo->pMetricMeta = ppMeta; } } @@ -3297,10 +2544,10 @@ int tscGetMetricMeta(SSqlObj *pSql, int32_t clauseIndex) { } for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - SMeterMetaInfo *pMMInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, i); + STableMetaInfo *pMMInfo = tscGetMetaInfo(pQueryInfo, i); - STableMeta *pMeterMeta = taosCacheAcquireByName(tscCacheHandle, pMMInfo->name); - tscAddMeterMetaInfo(pNewQueryInfo, pMMInfo->name, pMeterMeta, NULL, pMMInfo->numOfTags, pMMInfo->tagColumnIndex); + STableMeta *pTableMeta = taosCacheAcquireByName(tscCacheHandle, pMMInfo->name); + tscAddMeterMetaInfo(pNewQueryInfo, pMMInfo->name, pTableMeta, NULL, pMMInfo->numOfTags, pMMInfo->tagColumnIndex); } if ((code = tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { @@ -3327,42 +2574,17 @@ int tscGetMetricMeta(SSqlObj *pSql, int32_t clauseIndex) { // } tscTrace("%p allocate new pSqlObj:%p to get metricMeta", pSql, pNew); - if (pSql->fp == NULL) { - tsem_init(&pNew->rspSem, 0, 0); - tsem_init(&pNew->emptyRspSem, 0, 1); - - code = tscProcessSql(pNew); - - if (code == TSDB_CODE_SUCCESS) {//todo optimize the performance - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - char tagstr[TSDB_MAX_TAGS_LEN] = {0}; - - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, i); - tscGetMetricMetaCacheKey(pQueryInfo, tagstr, pMeterMetaInfo->pMeterMeta->uid); - -#ifdef _DEBUG_VIEW - printf("create metric key:%s, index:%d\n", tagstr, i); -#endif - - taosCacheRelease(tscCacheHandle, (void **)&(pMeterMetaInfo->pMetricMeta), false); - pMeterMetaInfo->pMetricMeta = (SSuperTableMeta *) taosCacheAcquireByName(tscCacheHandle, tagstr); - } - } - - tscFreeSqlObj(pNew); - } else { - pNew->fp = tscMeterMetaCallBack; - pNew->param = pSql; - code = tscProcessSql(pNew); - if (code == TSDB_CODE_SUCCESS) { - code = TSDB_CODE_ACTION_IN_PROGRESS; - } + pNew->fp = tscTableMetaCallBack; + pNew->param = pSql; + code = tscProcessSql(pNew); + if (code == TSDB_CODE_SUCCESS) { + code = TSDB_CODE_ACTION_IN_PROGRESS; } return code; } -void tscInitMsgs() { +void tscInitMsgsFp() { tscBuildMsg[TSDB_SQL_SELECT] = tscBuildQueryMsg; tscBuildMsg[TSDB_SQL_INSERT] = tscBuildSubmitMsg; tscBuildMsg[TSDB_SQL_FETCH] = tscBuildRetrieveMsg; @@ -3387,7 +2609,7 @@ void tscInitMsgs() { tscBuildMsg[TSDB_SQL_CONNECT] = tscBuildConnectMsg; tscBuildMsg[TSDB_SQL_USE_DB] = tscBuildUseDbMsg; - tscBuildMsg[TSDB_SQL_META] = tscBuildMeterMetaMsg; + tscBuildMsg[TSDB_SQL_META] = tscBuildTableMetaMsg; tscBuildMsg[TSDB_SQL_METRIC] = tscBuildMetricMetaMsg; tscBuildMsg[TSDB_SQL_MULTI_META] = tscBuildMultiMeterMetaMsg; @@ -3405,7 +2627,7 @@ void tscInitMsgs() { tscProcessMsgRsp[TSDB_SQL_DROP_TABLE] = tscProcessDropTableRsp; tscProcessMsgRsp[TSDB_SQL_CONNECT] = tscProcessConnectRsp; tscProcessMsgRsp[TSDB_SQL_USE_DB] = tscProcessUseDbRsp; - tscProcessMsgRsp[TSDB_SQL_META] = tscProcessMeterMetaRsp; + tscProcessMsgRsp[TSDB_SQL_META] = tscProcessTableMetaRsp; tscProcessMsgRsp[TSDB_SQL_METRIC] = tscProcessMetricMetaRsp; tscProcessMsgRsp[TSDB_SQL_MULTI_META] = tscProcessMultiMeterMetaRsp; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 930f054ff3984ed6228338e3ac562fb2516d4997..c3cc6ec9a825bce9e34c3bfcc1499cae4ac97954 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -15,49 +15,55 @@ #include "hash.h" #include "os.h" +#include "qast.h" #include "tcache.h" #include "tlog.h" #include "tnote.h" #include "trpc.h" -#include "tscJoinProcess.h" #include "tscProfile.h" #include "tscSecondaryMerge.h" +#include "tscSubquery.h" #include "tscUtil.h" #include "tsclient.h" #include "tscompression.h" #include "tsocket.h" #include "ttimer.h" -#include "tutil.h" #include "ttokendef.h" -#include "qast.h" +#include "tutil.h" -TAOS *taos_connect_imp(const char *ip, const char *user, const char *pass, const char *db, uint16_t port, - void (*fp)(void *, TAOS_RES *, int), void *param, void **taos) { - STscObj *pObj; +static bool validImpl(const char* str, size_t maxsize) { + if (str == NULL) { + return false; + } + + size_t len = strlen(str); + if (len <= 0 || len > maxsize) { + return false; + } + + return true; +} +static bool validUserName(const char* user) { + return validImpl(user, TSDB_USER_LEN); +} - taos_init(); +static bool validPassword(const char* passwd) { + return validImpl(passwd, TSDB_PASSWORD_LEN); +} - if (user == NULL) { +STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, const char *db, uint16_t port, + void (*fp)(void *, TAOS_RES *, int), void *param, void **taos) { + taos_init(); + + if (!validUserName(user)) { globalCode = TSDB_CODE_INVALID_ACCT; return NULL; - } else { - size_t len = strlen(user); - if (len <= 0 || len > TSDB_USER_LEN) { - globalCode = TSDB_CODE_INVALID_ACCT; - return NULL; - } } - if (pass == NULL) { + if (!validPassword(pass)) { globalCode = TSDB_CODE_INVALID_PASS; return NULL; - } else { - size_t len = strlen(pass); - if (len <= 0 || len > TSDB_KEY_LEN) { - globalCode = TSDB_CODE_INVALID_PASS; - return NULL; - } } if (tscInitRpc(user, pass) != 0) { @@ -83,14 +89,13 @@ TAOS *taos_connect_imp(const char *ip, const char *user, const char *pass, const } tscMgmtIpList.port = port ? port : tsMnodeShellPort; - - pObj = (STscObj *)malloc(sizeof(STscObj)); + + STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj)); if (NULL == pObj) { globalCode = TSDB_CODE_CLI_OUT_OF_MEMORY; return NULL; } - memset(pObj, 0, sizeof(STscObj)); pObj->signature = pObj; strncpy(pObj->user, user, TSDB_USER_LEN); @@ -115,18 +120,17 @@ TAOS *taos_connect_imp(const char *ip, const char *user, const char *pass, const pthread_mutex_init(&pObj->mutex, NULL); - SSqlObj *pSql = (SSqlObj *)malloc(sizeof(SSqlObj)); + SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); if (NULL == pSql) { globalCode = TSDB_CODE_CLI_OUT_OF_MEMORY; free(pObj); return NULL; } - memset(pSql, 0, sizeof(SSqlObj)); pSql->pTscObj = pObj; pSql->signature = pSql; tsem_init(&pSql->rspSem, 0, 0); - tsem_init(&pSql->emptyRspSem, 0, 1); +// tsem_init(&pSql->emptyRspSem, 0, 1); pObj->pSql = pSql; pSql->fp = fp; pSql->param = param; @@ -142,46 +146,69 @@ TAOS *taos_connect_imp(const char *ip, const char *user, const char *pass, const return NULL; } - pSql->res.code = tscProcessSql(pSql); - if (fp != NULL) { - tscTrace("%p DB async connection is opening", pObj); - return pObj; - } - - if (pSql->res.code) { - taos_close(pObj); - return NULL; - } - - tscTrace("%p DB connection is opened", pObj); return pObj; } +static void syncConnCallback(void *param, TAOS_RES *tres, int code) { + STscObj *pObj = (STscObj *)param; + assert(pObj != NULL && pObj->pSql != NULL); + + sem_post(&pObj->pSql->rspSem); +} + TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { if (ip == NULL || (ip != NULL && (strcmp("127.0.0.1", ip) == 0 || strcasecmp("localhost", ip) == 0))) { ip = tsMasterIp; } + tscTrace("try to create a connection to %s", ip); - void *taos = taos_connect_imp(ip, user, pass, db, port, NULL, NULL, NULL); - if (taos != NULL) { - STscObj *pObj = (STscObj *)taos; + STscObj *pObj = taosConnectImpl(ip, user, pass, db, port, NULL, NULL, NULL); + if (pObj != NULL) { + SSqlObj* pSql = pObj->pSql; + assert(pSql != NULL); + + pSql->fp = syncConnCallback; + pSql->param = pObj; + + tscProcessSql(pSql); + sem_wait(&pSql->rspSem); + + if (pSql->res.code != TSDB_CODE_SUCCESS) { + taos_close(pObj); + return NULL; + } + + tscTrace("%p DB connection is opening", pObj); // version compare only requires the first 3 segments of the version string - int code = taosCheckVersion(version, taos_get_server_info(taos), 3); + int code = taosCheckVersion(version, taos_get_server_info(pObj), 3); if (code != 0) { - pObj->pSql->res.code = code; - taos_close(taos); + pSql->res.code = code; + + taos_close(pObj); return NULL; + } else { + return pObj; } } - return taos; + return NULL; } TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, void **taos) { - return taos_connect_imp(ip, user, pass, db, port, fp, param, taos); + STscObj* pObj = taosConnectImpl(ip, user, pass, db, port, fp, param, taos); + if (pObj == NULL) { + return NULL; + } + + SSqlObj* pSql = pObj->pSql; + + pSql->res.code = tscProcessSql(pSql); + tscTrace("%p DB async connection is opening", pObj); + + return pObj; } void taos_close(TAOS *taos) { @@ -408,14 +435,14 @@ static char *getArithemicInputSrc(void *param, char *name, int32_t colId) { SSqlFunctionExpr * pExpr = pSupport->pExpr; int32_t index = -1; - for (int32_t i = 0; i < pExpr->pBinExprInfo.numOfCols; ++i) { - if (strcmp(name, pExpr->pBinExprInfo.pReqColumns[i].name) == 0) { + for (int32_t i = 0; i < pExpr->binExprInfo.numOfCols; ++i) { + if (strcmp(name, pExpr->binExprInfo.pReqColumns[i].name) == 0) { index = i; break; } } - assert(index >= 0 && index < pExpr->pBinExprInfo.numOfCols); + assert(index >= 0 && index < pExpr->binExprInfo.numOfCols); return pSupport->data[index] + pSupport->offset * pSupport->elemSize[index]; } @@ -465,21 +492,21 @@ static void **doSetResultRowData(SSqlObj *pSql) { sas->offset = 0; sas->pExpr = pQueryInfo->fieldsInfo.pExpr[i]; - sas->numOfCols = sas->pExpr->pBinExprInfo.numOfCols; + sas->numOfCols = sas->pExpr->binExprInfo.numOfCols; if (pRes->buffer[i] == NULL) { pRes->buffer[i] = malloc(tscFieldInfoGetField(pQueryInfo, i)->bytes); } for(int32_t k = 0; k < sas->numOfCols; ++k) { - int32_t columnIndex = sas->pExpr->pBinExprInfo.pReqColumns[k].colIdxInBuf; + int32_t columnIndex = sas->pExpr->binExprInfo.pReqColumns[k].colIdxInBuf; SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, columnIndex); sas->elemSize[k] = pExpr->resBytes; sas->data[k] = (pRes->data + pRes->numOfRows* pExpr->offset) + pRes->row*pExpr->resBytes; } - tSQLBinaryExprCalcTraverse(sas->pExpr->pBinExprInfo.pBinExpr, 1, pRes->buffer[i], sas, TSQL_SO_ASC, getArithemicInputSrc); + tSQLBinaryExprCalcTraverse(sas->pExpr->binExprInfo.pBinExpr, 1, pRes->buffer[i], sas, TSQL_SO_ASC, getArithemicInputSrc); pRes->tsrow[i] = pRes->buffer[i]; free(sas); //todo optimization @@ -509,7 +536,7 @@ static bool tscHashRemainDataInSubqueryResultSet(SSqlObj *pSql) { SSqlCmd *pCmd1 = &pSql->pSubs[i]->cmd; SQueryInfo * pQueryInfo1 = tscGetQueryInfoDetail(pCmd1, pCmd1->clauseIndex); - SMeterMetaInfo *pMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo1, 0); + STableMetaInfo *pMetaInfo = tscGetMetaInfo(pQueryInfo1, 0); assert(pQueryInfo1->numOfTables == 1); @@ -810,7 +837,7 @@ void taos_free_result_imp(TAOS_RES *res, int keepCmd) { pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE; - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); /* * case 1. Partial data have been retrieved from vnodes, but not all data has been retrieved yet. @@ -825,7 +852,7 @@ void taos_free_result_imp(TAOS_RES *res, int keepCmd) { if (pRes->code != TSDB_CODE_QUERY_CANCELLED && ((pRes->numOfRows > 0 && pCmd->command < TSDB_SQL_LOCAL) || (pRes->code == TSDB_CODE_SUCCESS && pRes->numOfRows == 0 && pCmd->command == TSDB_SQL_SELECT && - pSql->pStream == NULL && pMeterMetaInfo->pMeterMeta != NULL))) { + pSql->pStream == NULL && pTableMetaInfo->pTableMeta != NULL))) { pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; tscTrace("%p code:%d, numOfRows:%d, command:%d", pSql, pRes->code, pRes->numOfRows, pCmd->command); @@ -967,7 +994,7 @@ void taos_stop_query(TAOS_RES *res) { pSql->res.code = TSDB_CODE_QUERY_CANCELLED; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - if (tscIsTwoStageMergeMetricQuery(pQueryInfo, 0)) { + if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { tscKillMetricQuery(pSql); return; } @@ -1109,7 +1136,7 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t SQueryInfo *pQueryInfo = NULL; tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo); - SMeterMetaInfo *pMeterMetaInfo = tscAddEmptyMeterMetaInfo(pQueryInfo); + STableMetaInfo *pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); if ((code = tscAllocPayload(pCmd, tblListLen + 16)) != TSDB_CODE_SUCCESS) { return code; @@ -1144,7 +1171,7 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t return code; } - if ((code = setMeterID(pMeterMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) { + if ((code = setMeterID(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) { return code; } @@ -1154,7 +1181,7 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t return code; } - if (payloadLen + strlen(pMeterMetaInfo->name) + 128 >= pCmd->allocSize) { + if (payloadLen + strlen(pTableMetaInfo->name) + 128 >= pCmd->allocSize) { char *pNewMem = realloc(pCmd->payload, pCmd->allocSize + tblListLen); if (pNewMem == NULL) { code = TSDB_CODE_CLI_OUT_OF_MEMORY; @@ -1167,7 +1194,7 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t pMsg = pCmd->payload; } - payloadLen += sprintf(pMsg + payloadLen, "%s,", pMeterMetaInfo->name); + payloadLen += sprintf(pMsg + payloadLen, "%s,", pTableMetaInfo->name); } *(pMsg + payloadLen) = '\0'; diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index d3ceb5ee84cb3987d487a30aaba4ae1f2cd7c663..46e3ac2e60b9e28fa21251c0fbe4a2db3fdd0351 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -13,15 +13,16 @@ * along with this program. If not, see . */ +#include #include "os.h" +#include "taosmsg.h" #include "tlog.h" +#include "tscUtil.h" +#include "tsched.h" +#include "tsclient.h" #include "ttime.h" #include "ttimer.h" #include "tutil.h" -#include "tsched.h" -#include "taosmsg.h" -#include "tscUtil.h" -#include "tsclient.h" #include "tscProfile.h" @@ -71,14 +72,14 @@ static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) { pSql->param = pStream; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - int code = tscGetMeterMeta(pSql, pMeterMetaInfo); + int code = tscGetTableMeta(pSql, pTableMetaInfo); pSql->res.code = code; if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; - if (code == 0 && UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) { + if (code == 0 && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { code = tscGetMetricMeta(pSql, 0); pSql->res.code = code; @@ -96,7 +97,7 @@ static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) { return; } - tscTrace("%p stream:%p start stream query on:%s", pSql, pStream, pMeterMetaInfo->name); + tscTrace("%p stream:%p start stream query on:%s", pSql, pStream, pTableMetaInfo->name); tscProcessSql(pStream->pSql); tscIncStreamExecutionCount(pStream); @@ -145,8 +146,8 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf tscError("%p stream:%p, query data failed, code:%d, retry in %" PRId64 "ms", pStream->pSql, pStream, numOfRows, retryDelay); - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(&pStream->pSql->cmd, 0, 0); - tscClearMeterMetaInfo(pMeterMetaInfo, true); + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0, 0); + tscClearMeterMetaInfo(pTableMetaInfo, true); tscSetRetryTimer(pStream, pStream->pSql, retryDelay); return; @@ -171,12 +172,12 @@ static void tscSetTimestampForRes(SSqlStream *pStream, SSqlObj *pSql) { static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOfRows) { SSqlStream * pStream = (SSqlStream *)param; SSqlObj * pSql = (SSqlObj *)res; - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, 0, 0); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); if (pSql == NULL || numOfRows < 0) { int64_t retryDelayTime = tscGetRetryDelayTime(pStream->slidingTime, pStream->precision); tscError("%p stream:%p, retrieve data failed, code:%d, retry in %" PRId64 "ms", pSql, pStream, numOfRows, retryDelayTime); - tscClearMeterMetaInfo(pMeterMetaInfo, true); + tscClearMeterMetaInfo(pTableMetaInfo, true); tscSetRetryTimer(pStream, pStream->pSql, retryDelayTime); return; @@ -254,11 +255,11 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf } } - tscTrace("%p stream:%p, query on:%s, fetch result completed, fetched rows:%d", pSql, pStream, pMeterMetaInfo->name, + tscTrace("%p stream:%p, query on:%s, fetch result completed, fetched rows:%d", pSql, pStream, pTableMetaInfo->name, pStream->numOfRes); // release the metric/meter meta information reference, so data in cache can be updated - tscClearMeterMetaInfo(pMeterMetaInfo, false); + tscClearMeterMetaInfo(pTableMetaInfo, false); tscSetNextLaunchTimer(pStream, pSql); } } @@ -539,13 +540,14 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p } SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); + pStream->fp = fp; pStream->callback = callback; pStream->param = param; pStream->pSql = pSql; - pStream->precision = pMeterMetaInfo->pMeterMeta->precision; + pStream->precision = tinfo.precision; pStream->ctime = taosGetTimestamp(pStream->precision); pStream->etime = pQueryInfo->etime; @@ -560,7 +562,7 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p taosTmrReset(tscProcessStreamTimer, starttime, pStream, tscTmr, &pStream->pTimer); tscTrace("%p stream:%p is opened, query on:%s, interval:%" PRId64 ", sliding:%" PRId64 ", first launched in:%" PRId64 ", sql:%s", pSql, - pStream, pMeterMetaInfo->name, pStream->interval, pStream->slidingTime, starttime, sqlstr); + pStream, pTableMetaInfo->name, pStream->interval, pStream->slidingTime, starttime, sqlstr); return pStream; } diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index f14643a72fb2afc97a50239529ae0678c55cd208..ceb3b180b90a9d6281b09102f844373eb641286b 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -175,10 +175,10 @@ int tscUpdateSubscription(STscObj* pObj, SSub* pSub) { return 0; } - SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0, 0); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0, 0); int numOfTables = 0; - if (!UTIL_METER_IS_NOMRAL_METER(pMeterMetaInfo)) { - SSuperTableMeta* pMetricMeta = pMeterMetaInfo->pMetricMeta; + if (!UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { + SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta; for (int32_t i = 0; i < pMetricMeta->numOfVnodes; i++) { SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, i); numOfTables += pVnodeSidList->numOfSids; @@ -191,19 +191,19 @@ int tscUpdateSubscription(STscObj* pObj, SSub* pSub) { return 0; } - if (UTIL_METER_IS_NOMRAL_METER(pMeterMetaInfo)) { + if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { numOfTables = 1; - int64_t uid = pMeterMetaInfo->pMeterMeta->uid; + int64_t uid = pTableMetaInfo->pTableMeta->uid; progress[0].uid = uid; progress[0].key = tscGetSubscriptionProgress(pSub, uid); } else { - SSuperTableMeta* pMetricMeta = pMeterMetaInfo->pMetricMeta; + SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta; numOfTables = 0; for (int32_t i = 0; i < pMetricMeta->numOfVnodes; i++) { SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, i); for (int32_t j = 0; j < pVnodeSidList->numOfSids; j++) { - STableSidExtInfo *pMeterInfo = tscGetMeterSidInfo(pVnodeSidList, j); - int64_t uid = pMeterInfo->uid; + STableSidExtInfo *pTableMetaInfo = tscGetMeterSidInfo(pVnodeSidList, j); + int64_t uid = pTableMetaInfo->uid; progress[numOfTables].uid = uid; progress[numOfTables++].key = tscGetSubscriptionProgress(pSub, uid); } @@ -385,7 +385,7 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) { pSql->cmd.command = TSDB_SQL_SELECT; pQueryInfo->type = type; - tscGetMeterMetaInfo(&pSql->cmd, 0, 0)->vnodeIndex = 0; + tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0)->vnodeIndex = 0; } tscDoQuery(pSql); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c new file mode 100644 index 0000000000000000000000000000000000000000..1df977d99626cec34ac3750eb668244dd06c33d4 --- /dev/null +++ b/src/client/src/tscSubquery.c @@ -0,0 +1,1572 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tscSubquery.h" +#include "os.h" +#include "qtsbuf.h" +#include "tsclient.h" + +typedef struct SInsertSupporter { + SSubqueryState* pState; + SSqlObj* pSql; +} SInsertSupporter; + +static void freeSubqueryObj(SSqlObj* pSql); + +static bool doCompare(int32_t order, int64_t left, int64_t right) { + if (order == TSQL_SO_ASC) { + return left < right; + } else { + return left > right; + } +} + +static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSubquerySupporter* pSupporter1, + SJoinSubquerySupporter* pSupporter2, TSKEY* st, TSKEY* et) { + STSBuf* output1 = tsBufCreate(true); + STSBuf* output2 = tsBufCreate(true); + + *st = INT64_MAX; + *et = INT64_MIN; + + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); + + SLimitVal* pLimit = &pQueryInfo->limit; + int32_t order = pQueryInfo->order.order; + + SQueryInfo* pSubQueryInfo1 = tscGetQueryInfoDetail(&pSql->pSubs[0]->cmd, 0); + SQueryInfo* pSubQueryInfo2 = tscGetQueryInfoDetail(&pSql->pSubs[1]->cmd, 0); + + pSubQueryInfo1->tsBuf = output1; + pSubQueryInfo2->tsBuf = output2; + + tsBufResetPos(pSupporter1->pTSBuf); + tsBufResetPos(pSupporter2->pTSBuf); + + // TODO add more details information + if (!tsBufNextPos(pSupporter1->pTSBuf)) { + tsBufFlush(output1); + tsBufFlush(output2); + + tscTrace("%p input1 is empty, 0 for secondary query after ts blocks intersecting", pSql); + return 0; + } + + if (!tsBufNextPos(pSupporter2->pTSBuf)) { + tsBufFlush(output1); + tsBufFlush(output2); + + tscTrace("%p input2 is empty, 0 for secondary query after ts blocks intersecting", pSql); + return 0; + } + + int64_t numOfInput1 = 1; + int64_t numOfInput2 = 1; + + while (1) { + STSElem elem1 = tsBufGetElem(pSupporter1->pTSBuf); + STSElem elem2 = tsBufGetElem(pSupporter2->pTSBuf); + +#ifdef _DEBUG_VIEW + // for debug purpose + tscPrint("%" PRId64 ", tags:%d \t %" PRId64 ", tags:%d", elem1.ts, elem1.tag, elem2.ts, elem2.tag); +#endif + + if (elem1.tag < elem2.tag || (elem1.tag == elem2.tag && doCompare(order, elem1.ts, elem2.ts))) { + if (!tsBufNextPos(pSupporter1->pTSBuf)) { + break; + } + + numOfInput1++; + } else if (elem1.tag > elem2.tag || (elem1.tag == elem2.tag && doCompare(order, elem2.ts, elem1.ts))) { + if (!tsBufNextPos(pSupporter2->pTSBuf)) { + break; + } + + numOfInput2++; + } else { + /* + * in case of stable query, limit/offset is not applied here. the limit/offset is applied to the + * final results which is acquired after the secondry merge of in the client. + */ + if (pLimit->offset == 0 || pQueryInfo->intervalTime > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) { + if (*st > elem1.ts) { + *st = elem1.ts; + } + + if (*et < elem1.ts) { + *et = elem1.ts; + } + + tsBufAppend(output1, elem1.vnode, elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts)); + tsBufAppend(output2, elem2.vnode, elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts)); + } else { + pLimit->offset -= 1; + } + + if (!tsBufNextPos(pSupporter1->pTSBuf)) { + break; + } + + numOfInput1++; + + if (!tsBufNextPos(pSupporter2->pTSBuf)) { + break; + } + + numOfInput2++; + } + } + + /* + * failed to set the correct ts order yet in two cases: + * 1. only one element + * 2. only one element for each tag. + */ + if (output1->tsOrder == -1) { + output1->tsOrder = TSQL_SO_ASC; + output2->tsOrder = TSQL_SO_ASC; + } + + tsBufFlush(output1); + tsBufFlush(output2); + + tsBufDestory(pSupporter1->pTSBuf); + tsBufDestory(pSupporter2->pTSBuf); + + tscTrace("%p input1:%" PRId64 ", input2:%" PRId64 ", final:%" PRId64 " for secondary query after ts blocks " + "intersecting, skey:%" PRId64 ", ekey:%" PRId64, pSql, + numOfInput1, numOfInput2, output1->numOfTotal, *st, *et); + + return output1->numOfTotal; +} + +// todo handle failed to create sub query +SJoinSubquerySupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, int32_t index) { + SJoinSubquerySupporter* pSupporter = calloc(1, sizeof(SJoinSubquerySupporter)); + if (pSupporter == NULL) { + return NULL; + } + + pSupporter->pObj = pSql; + pSupporter->pState = pState; + + pSupporter->subqueryIndex = index; + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); + + pSupporter->interval = pQueryInfo->intervalTime; + pSupporter->limit = pQueryInfo->limit; + + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, index); + pSupporter->uid = pTableMetaInfo->pTableMeta->uid; + + assert (pSupporter->uid != 0); + + getTmpfilePath("join-", pSupporter->path); + pSupporter->f = fopen(pSupporter->path, "w"); + + if (pSupporter->f == NULL) { + tscError("%p failed to create tmp file:%s, reason:%s", pSql, pSupporter->path, strerror(errno)); + } + + return pSupporter; +} + +void tscDestroyJoinSupporter(SJoinSubquerySupporter* pSupporter) { + if (pSupporter == NULL) { + return; + } + + tscSqlExprInfoDestroy(&pSupporter->exprsInfo); + tscColumnBaseInfoDestroy(&pSupporter->colList); + + tscClearFieldInfo(&pSupporter->fieldsInfo); + + if (pSupporter->f != NULL) { + fclose(pSupporter->f); + unlink(pSupporter->path); + } + + tscTagCondRelease(&pSupporter->tagCond); + free(pSupporter); +} + +/* + * need the secondary query process + * In case of count(ts)/count(*)/spread(ts) query, that are only applied to + * primary timestamp column , the secondary query is not necessary + * + */ +bool needSecondaryQuery(SQueryInfo* pQueryInfo) { + for (int32_t i = 0; i < pQueryInfo->colList.numOfCols; ++i) { + SColumnBase* pBase = tscColumnBaseInfoGet(&pQueryInfo->colList, i); + if (pBase->colIndex.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + return true; + } + } + + return false; +} + +/* + * launch secondary stage query to fetch the result that contains timestamp in set + */ +int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { + int32_t numOfSub = 0; + SJoinSubquerySupporter* pSupporter = NULL; + + /* + * If the columns are not involved in the final select clause, the secondary query will not be launched + * for the subquery. + */ + SSubqueryState* pState = NULL; + + for (int32_t i = 0; i < pSql->numOfSubs; ++i) { + pSupporter = pSql->pSubs[i]->param; + if (pSupporter->exprsInfo.numOfExprs > 0) { + ++numOfSub; + } + } + + assert(numOfSub > 0); + + // scan all subquery, if one sub query has only ts, ignore it + tscTrace("%p start to launch secondary subqueries, total:%d, only:%d needs to query, others are not retrieve in " + "select clause", pSql, pSql->numOfSubs, numOfSub); + + /* + * the subqueries that do not actually launch the secondary query to virtual node is set as completed. + */ + pState = pSupporter->pState; + pState->numOfTotal = pSql->numOfSubs; + pState->numOfCompleted = (pSql->numOfSubs - numOfSub); + + bool success = true; + + for (int32_t i = 0; i < pSql->numOfSubs; ++i) { + SSqlObj *pPrevSub = pSql->pSubs[i]; + pSql->pSubs[i] = NULL; + + pSupporter = pPrevSub->param; + + if (pSupporter->exprsInfo.numOfExprs == 0) { + tscTrace("%p subIndex: %d, not need to launch query, ignore it", pSql, i); + + tscDestroyJoinSupporter(pSupporter); + tscFreeSqlObj(pPrevSub); + + pSql->pSubs[i] = NULL; + continue; + } + + SQueryInfo *pSubQueryInfo = tscGetQueryInfoDetail(&pPrevSub->cmd, 0); + STSBuf *pTSBuf = pSubQueryInfo->tsBuf; + pSubQueryInfo->tsBuf = NULL; + + // free result for async object will also free sqlObj + assert(pSubQueryInfo->exprsInfo.numOfExprs == 1); // ts_comp query only requires one resutl columns + taos_free_result(pPrevSub); + + SSqlObj *pNew = createSubqueryObj(pSql, (int16_t) i, tscJoinQueryCallback, pSupporter, NULL); + if (pNew == NULL) { + tscDestroyJoinSupporter(pSupporter); + success = false; + break; + } + + tscClearSubqueryInfo(&pNew->cmd); + pSql->pSubs[i] = pNew; + + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); + pQueryInfo->tsBuf = pTSBuf; // transfer the ownership of timestamp comp-z data to the new created object + + // set the second stage sub query for join process + pQueryInfo->type |= TSDB_QUERY_TYPE_JOIN_SEC_STAGE; + + pQueryInfo->intervalTime = pSupporter->interval; + pQueryInfo->groupbyExpr = pSupporter->groupbyExpr; + + tscColumnBaseInfoCopy(&pQueryInfo->colList, &pSupporter->colList, 0); + tscTagCondCopy(&pQueryInfo->tagCond, &pSupporter->tagCond); + + tscSqlExprCopy(&pQueryInfo->exprsInfo, &pSupporter->exprsInfo, pSupporter->uid, false); + tscFieldInfoCopyAll(&pQueryInfo->fieldsInfo, &pSupporter->fieldsInfo); + + pSupporter->exprsInfo.numOfExprs = 0; + pSupporter->fieldsInfo.numOfOutputCols = 0; + + /* + * if the first column of the secondary query is not ts function, add this function. + * Because this column is required to filter with timestamp after intersecting. + */ + if (pSupporter->exprsInfo.pExprs[0]->functionId != TSDB_FUNC_TS) { + tscAddTimestampColumn(pQueryInfo, TSDB_FUNC_TS, 0); + } + + SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); + assert(pNew->numOfSubs == 0 && pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1); + + tscFieldInfoCalOffset(pNewQueryInfo); + + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0); + + /* + * When handling the projection query, the offset value will be modified for table-table join, which is changed + * during the timestamp intersection. + */ + pSupporter->limit = pQueryInfo->limit; + pNewQueryInfo->limit = pSupporter->limit; + + // fetch the join tag column + if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + SSqlExpr *pExpr = tscSqlExprGet(pNewQueryInfo, 0); + assert(pQueryInfo->tagCond.joinInfo.hasJoin); + + int16_t tagColIndex = tscGetJoinTagColIndexByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid); + pExpr->param[0].i64Key = tagColIndex; + pExpr->numOfParams = 1; + } + + tscPrintSelectClause(pNew, 0); + + tscTrace("%p subquery:%p tableIndex:%d, vnodeIdx:%d, type:%d, exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s", + pSql, pNew, 0, pTableMetaInfo->vnodeIndex, pNewQueryInfo->type, + pNewQueryInfo->exprsInfo.numOfExprs, pNewQueryInfo->colList.numOfCols, + pNewQueryInfo->fieldsInfo.numOfOutputCols, pNewQueryInfo->pTableMetaInfo[0]->name); + } + + //prepare the subqueries object failed, abort + if (!success) { + pSql->res.code = TSDB_CODE_CLI_OUT_OF_MEMORY; + tscError("%p failed to prepare subqueries objs for secondary phase query, numOfSub:%d, code:%d", pSql, + pSql->numOfSubs, pSql->res.code); + freeSubqueryObj(pSql); + + return pSql->res.code; + } + + for(int32_t i = 0; i < pSql->numOfSubs; ++i) { + SSqlObj* pSub = pSql->pSubs[i]; + if (pSub == NULL) { + continue; + } + + tscProcessSql(pSub); + } + + return TSDB_CODE_SUCCESS; +} + +void freeSubqueryObj(SSqlObj* pSql) { + SSubqueryState* pState = NULL; + + for (int32_t i = 0; i < pSql->numOfSubs; ++i) { + if (pSql->pSubs[i] != NULL) { + SJoinSubquerySupporter* p = pSql->pSubs[i]->param; + pState = p->pState; + + tscDestroyJoinSupporter(p); + + if (pSql->pSubs[i]->res.code == TSDB_CODE_SUCCESS) { + taos_free_result(pSql->pSubs[i]); + } + } + } + + tfree(pState); + pSql->numOfSubs = 0; +} + +static void doQuitSubquery(SSqlObj* pParentSql) { + freeSubqueryObj(pParentSql); + + tsem_wait(&pParentSql->emptyRspSem); + tsem_wait(&pParentSql->emptyRspSem); + + tsem_post(&pParentSql->rspSem); +} + +static void quitAllSubquery(SSqlObj* pSqlObj, SJoinSubquerySupporter* pSupporter) { + int32_t numOfTotal = pSupporter->pState->numOfTotal; + int32_t finished = atomic_add_fetch_32(&pSupporter->pState->numOfCompleted, 1); + + if (finished >= numOfTotal) { + pSqlObj->res.code = abs(pSupporter->pState->code); + tscError("%p all subquery return and query failed, global code:%d", pSqlObj, pSqlObj->res.code); + + doQuitSubquery(pSqlObj); + } +} + +// update the query time range according to the join results on timestamp +static void updateQueryTimeRange(SQueryInfo* pQueryInfo, int64_t st, int64_t et) { + assert(pQueryInfo->stime <= st && pQueryInfo->etime >= et); + + pQueryInfo->stime = st; + pQueryInfo->etime = et; +} + +static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { + SJoinSubquerySupporter* pSupporter = (SJoinSubquerySupporter*)param; + SSqlObj* pParentSql = pSupporter->pObj; + + SSqlObj* pSql = (SSqlObj*)tres; + SSqlCmd* pCmd = &pSql->cmd; + SSqlRes* pRes = &pSql->res; + + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + + if ((pQueryInfo->type & TSDB_QUERY_TYPE_JOIN_SEC_STAGE) == 0) { + if (pSupporter->pState->code != TSDB_CODE_SUCCESS) { + tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, numOfRows, + pSupporter->pState->code); + + quitAllSubquery(pParentSql, pSupporter); + return; + } + + if (numOfRows > 0) { // write the data into disk + fwrite(pSql->res.data, pSql->res.numOfRows, 1, pSupporter->f); + fclose(pSupporter->f); + + STSBuf* pBuf = tsBufCreateFromFile(pSupporter->path, true); + if (pBuf == NULL) { + tscError("%p invalid ts comp file from vnode, abort sub query, file size:%d", pSql, numOfRows); + + pSupporter->pState->code = TSDB_CODE_APP_ERROR; // todo set the informative code + quitAllSubquery(pParentSql, pSupporter); + return; + } + + if (pSupporter->pTSBuf == NULL) { + tscTrace("%p create tmp file for ts block:%s", pSql, pBuf->path); + pSupporter->pTSBuf = pBuf; + } else { + assert(pQueryInfo->numOfTables == 1); // for subquery, only one metermetaInfo + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + tsBufMerge(pSupporter->pTSBuf, pBuf, pTableMetaInfo->vnodeIndex); + tsBufDestory(pBuf); + } + + // open new file to save the result + getTmpfilePath("ts-join", pSupporter->path); + pSupporter->f = fopen(pSupporter->path, "w"); + pSql->res.row = pSql->res.numOfRows; + + taos_fetch_rows_a(tres, joinRetrieveCallback, param); + } else if (numOfRows == 0) { // no data from this vnode anymore + SQueryInfo* pParentQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, pParentSql->cmd.clauseIndex); + + //todo refactor + if (tscNonOrderedProjectionQueryOnSTable(pParentQueryInfo, 0)) { + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + assert(pQueryInfo->numOfTables == 1); + + // for projection query, need to try next vnode + int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes; + if ((++pTableMetaInfo->vnodeIndex) < totalVnode) { + tscTrace("%p current vnode:%d exhausted, try next:%d. total vnode:%d. current numOfRes:%d", pSql, + pTableMetaInfo->vnodeIndex - 1, pTableMetaInfo->vnodeIndex, totalVnode, pRes->numOfTotal); + + pSql->cmd.command = TSDB_SQL_SELECT; + pSql->fp = tscJoinQueryCallback; + tscProcessSql(pSql); + + return; + } + } + + int32_t numOfTotal = pSupporter->pState->numOfTotal; + int32_t finished = atomic_add_fetch_32(&pSupporter->pState->numOfCompleted, 1); + + if (finished >= numOfTotal) { + assert(finished == numOfTotal); + + if (pSupporter->pState->code != TSDB_CODE_SUCCESS) { + tscTrace("%p sub:%p, numOfSub:%d, quit from further procedure due to other queries failure", pParentSql, tres, + pSupporter->subqueryIndex); + doQuitSubquery(pParentSql); + return; + } + + tscTrace("%p all subqueries retrieve ts complete, do ts block intersect", pParentSql); + + SJoinSubquerySupporter* p1 = pParentSql->pSubs[0]->param; + SJoinSubquerySupporter* p2 = pParentSql->pSubs[1]->param; + + TSKEY st, et; + + int64_t num = doTSBlockIntersect(pParentSql, p1, p2, &st, &et); + if (num <= 0) { // no result during ts intersect + tscTrace("%p free all sub SqlObj and quit", pParentSql); + doQuitSubquery(pParentSql); + } else { + updateQueryTimeRange(pParentQueryInfo, st, et); + tscLaunchSecondPhaseSubqueries(pParentSql); + } + } + } else { // failure of sub query + tscError("%p sub query failed, code:%d, index:%d", pSql, numOfRows, pSupporter->subqueryIndex); + pSupporter->pState->code = numOfRows; + + quitAllSubquery(pParentSql, pSupporter); + return; + } + + } else { // secondary stage retrieve, driven by taos_fetch_row or other functions + if (numOfRows < 0) { + pSupporter->pState->code = numOfRows; + tscError("%p retrieve failed, code:%d, index:%d", pSql, numOfRows, pSupporter->subqueryIndex); + } + + if (numOfRows >= 0) { + pSql->res.numOfTotal += pSql->res.numOfRows; + } + + if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && numOfRows == 0) { + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + assert(pQueryInfo->numOfTables == 1); + + // for projection query, need to try next vnode if current vnode is exhausted + if ((++pTableMetaInfo->vnodeIndex) < pTableMetaInfo->pMetricMeta->numOfVnodes) { + pSupporter->pState->numOfCompleted = 0; + pSupporter->pState->numOfTotal = 1; + + pSql->cmd.command = TSDB_SQL_SELECT; + pSql->fp = tscJoinQueryCallback; + tscProcessSql(pSql); + + return; + } + } + + int32_t numOfTotal = pSupporter->pState->numOfTotal; + int32_t finished = atomic_add_fetch_32(&pSupporter->pState->numOfCompleted, 1); + + if (finished >= numOfTotal) { + assert(finished == numOfTotal); + tscTrace("%p all %d secondary subquery retrieves completed, global code:%d", tres, numOfTotal, + pParentSql->res.code); + + if (pSupporter->pState->code != TSDB_CODE_SUCCESS) { + pParentSql->res.code = abs(pSupporter->pState->code); + freeSubqueryObj(pParentSql); + } + + tsem_post(&pParentSql->rspSem); + } else { + tscTrace("%p sub:%p completed, completed:%d, total:%d", pParentSql, tres, finished, numOfTotal); + } + } +} + +static SJoinSubquerySupporter* tscUpdateSubqueryStatus(SSqlObj* pSql, int32_t numOfFetch) { + int32_t notInvolved = 0; + SJoinSubquerySupporter* pSupporter = NULL; + SSubqueryState* pState = NULL; + + for(int32_t i = 0; i < pSql->numOfSubs; ++i) { + if (pSql->pSubs[i] == NULL) { + notInvolved++; + } else { + pSupporter = (SJoinSubquerySupporter*)pSql->pSubs[i]->param; + pState = pSupporter->pState; + } + } + + pState->numOfTotal = pSql->numOfSubs; + pState->numOfCompleted = pSql->numOfSubs - numOfFetch; + + return pSupporter; +} + +void tscFetchDatablockFromSubquery(SSqlObj* pSql) { + int32_t numOfFetch = 0; + assert(pSql->numOfSubs >= 1); + + for (int32_t i = 0; i < pSql->numOfSubs; ++i) { + if (pSql->pSubs[i] == NULL) { // this subquery does not need to involve in secondary query + continue; + } + + SSqlRes *pRes = &pSql->pSubs[i]->res; + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->pSubs[i]->cmd, 0); + + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { + if (pRes->row >= pRes->numOfRows && pTableMetaInfo->vnodeIndex < pTableMetaInfo->pMetricMeta->numOfVnodes && + (!tscHasReachLimitation(pQueryInfo, pRes))) { + numOfFetch++; + } + } else { + if (pRes->row >= pRes->numOfRows && (!tscHasReachLimitation(pQueryInfo, pRes))) { + numOfFetch++; + } + } + } + + if (numOfFetch <= 0) { + return; + } + + // TODO multi-vnode retrieve for projection query with limitation has bugs, since the global limiation is not handled + tscTrace("%p retrieve data from %d subqueries", pSql, numOfFetch); + + SJoinSubquerySupporter* pSupporter = tscUpdateSubqueryStatus(pSql, numOfFetch); + + for (int32_t i = 0; i < pSql->numOfSubs; ++i) { + SSqlObj* pSql1 = pSql->pSubs[i]; + if (pSql1 == NULL) { + continue; + } + + SSqlRes* pRes1 = &pSql1->res; + SSqlCmd* pCmd1 = &pSql1->cmd; + + pSupporter = (SJoinSubquerySupporter*)pSql1->param; + + // wait for all subqueries completed + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd1, 0); + assert(pRes1->numOfRows >= 0 && pQueryInfo->numOfTables == 1); + + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + if (pRes1->row >= pRes1->numOfRows) { + tscTrace("%p subquery:%p retrieve data from vnode, subquery:%d, vnodeIndex:%d", pSql, pSql1, + pSupporter->subqueryIndex, pTableMetaInfo->vnodeIndex); + + tscResetForNextRetrieve(pRes1); + pSql1->fp = joinRetrieveCallback; + + if (pCmd1->command < TSDB_SQL_LOCAL) { + pCmd1->command = (pCmd1->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; + } + + tscProcessSql(pSql1); + } + } + + // wait for all subquery completed + tsem_wait(&pSql->rspSem); + + // update the records for each subquery + for(int32_t i = 0; i < pSql->numOfSubs; ++i) { + if (pSql->pSubs[i] == NULL) { + continue; + } + + SSqlRes* pRes1 = &pSql->pSubs[i]->res; + pRes1->numOfTotalInCurrentClause += pRes1->numOfRows; + } +} + +// all subqueries return, set the result output index +void tscSetupOutputColumnIndex(SSqlObj* pSql) { + SSqlCmd* pCmd = &pSql->cmd; + SSqlRes* pRes = &pSql->res; + + tscTrace("%p all subquery response, retrieve data", pSql); + + if (pRes->pColumnIndex != NULL) { + return; // the column transfer support struct has been built + } + + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + pRes->pColumnIndex = calloc(1, sizeof(SColumnIndex) * pQueryInfo->fieldsInfo.numOfOutputCols); + + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + + int32_t tableIndexOfSub = -1; + for (int32_t j = 0; j < pQueryInfo->numOfTables; ++j) { + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, j); + if (pTableMetaInfo->pTableMeta->uid == pExpr->uid) { + tableIndexOfSub = j; + break; + } + } + + assert(tableIndexOfSub >= 0 && tableIndexOfSub < pQueryInfo->numOfTables); + + SSqlCmd* pSubCmd = &pSql->pSubs[tableIndexOfSub]->cmd; + SQueryInfo* pSubQueryInfo = tscGetQueryInfoDetail(pSubCmd, 0); + + for (int32_t k = 0; k < pSubQueryInfo->exprsInfo.numOfExprs; ++k) { + SSqlExpr* pSubExpr = tscSqlExprGet(pSubQueryInfo, k); + if (pExpr->functionId == pSubExpr->functionId && pExpr->colInfo.colId == pSubExpr->colInfo.colId) { + pRes->pColumnIndex[i] = (SColumnIndex){.tableIndex = tableIndexOfSub, .columnIndex = k}; + break; + } + } + } +} + +void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { + SSqlObj* pSql = (SSqlObj*)tres; + // STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); + + // int32_t idx = pSql->cmd.vnodeIdx; + + // SVnodeSidList *vnodeInfo = NULL; + // if (pTableMetaInfo->pMetricMeta != NULL) { + // vnodeInfo = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx - 1); + // } + + SJoinSubquerySupporter* pSupporter = (SJoinSubquerySupporter*)param; + + // if (atomic_add_fetch_32(pSupporter->numOfComplete, 1) >= + // pSupporter->numOfTotal) { + // SSqlObj *pParentObj = pSupporter->pObj; + // + // if ((pSql->cmd.type & TSDB_QUERY_TYPE_JOIN_SEC_STAGE) != 1) { + // int32_t num = 0; + // tscFetchDatablockFromSubquery(pParentObj); + // TSKEY* ts = tscGetQualifiedTSList(pParentObj, &num); + // + // if (num <= 0) { + // // no qualified result + // } + // + // tscLaunchSecondPhaseSubqueries(pSql, ts, num); + // } else { + + // } + // } else { + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); + if ((pQueryInfo->type & TSDB_QUERY_TYPE_JOIN_SEC_STAGE) != TSDB_QUERY_TYPE_JOIN_SEC_STAGE) { + if (code != TSDB_CODE_SUCCESS) { // direct call joinRetrieveCallback and set the error code + joinRetrieveCallback(param, pSql, code); + } else { // first stage query, continue to retrieve data + pSql->fp = joinRetrieveCallback; + pSql->cmd.command = TSDB_SQL_FETCH; + tscProcessSql(pSql); + } + + } else { // second stage join subquery + SSqlObj* pParentSql = pSupporter->pObj; + + if (pSupporter->pState->code != TSDB_CODE_SUCCESS) { + tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, code, + pSupporter->pState->code); + quitAllSubquery(pParentSql, pSupporter); + + return; + } + + if (code != TSDB_CODE_SUCCESS) { + tscError("%p sub query failed, code:%d, set global code:%d, index:%d", pSql, code, code, + pSupporter->subqueryIndex); + pSupporter->pState->code = code; // todo set the informative code + + quitAllSubquery(pParentSql, pSupporter); + } else { + int32_t numOfTotal = pSupporter->pState->numOfTotal; + int32_t finished = atomic_add_fetch_32(&pSupporter->pState->numOfCompleted, 1); + + if (finished >= numOfTotal) { + assert(finished == numOfTotal); + + tscSetupOutputColumnIndex(pParentSql); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + /** + * if the query is a continue query (vnodeIndex > 0 for projection query) for next vnode, do the retrieval of + * data instead of returning to its invoker + */ + if (pTableMetaInfo->vnodeIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { + assert(pTableMetaInfo->vnodeIndex < pTableMetaInfo->pMetricMeta->numOfVnodes); + pSupporter->pState->numOfCompleted = 0; // reset the record value + + pSql->fp = joinRetrieveCallback; // continue retrieve data + pSql->cmd.command = TSDB_SQL_FETCH; + tscProcessSql(pSql); + } else { // first retrieve from vnode during the secondary stage sub-query + if (pParentSql->fp == NULL) { + tsem_wait(&pParentSql->emptyRspSem); + tsem_wait(&pParentSql->emptyRspSem); + + tsem_post(&pParentSql->rspSem); + } else { + // set the command flag must be after the semaphore been correctly set. + // pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC; + // if (pPObj->res.code == TSDB_CODE_SUCCESS) { + // (*pPObj->fp)(pPObj->param, pPObj, 0); + // } else { + // tscQueueAsyncRes(pPObj); + // } + assert(0); + } + } + } + } + } +} + +///////////////////////////////////////////////////////////////////////////////////////// +static void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code); + +static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj); + +// todo merge with callback +int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSubquerySupporter *pSupporter) { + SSqlCmd * pCmd = &pSql->cmd; + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + + pSql->res.qhandle = 0x1; + pSql->res.numOfRows = 0; + + if (pSql->pSubs == NULL) { + pSql->pSubs = calloc(pSupporter->pState->numOfTotal, POINTER_BYTES); + if (pSql->pSubs == NULL) { + return TSDB_CODE_CLI_OUT_OF_MEMORY; + } + } + + SSqlObj *pNew = createSubqueryObj(pSql, tableIndex, tscJoinQueryCallback, pSupporter, NULL); + if (pNew == NULL) { + return TSDB_CODE_CLI_OUT_OF_MEMORY; + } + + pSql->pSubs[pSql->numOfSubs++] = pNew; + assert(pSql->numOfSubs <= pSupporter->pState->numOfTotal); + + if (QUERY_IS_JOIN_QUERY(pQueryInfo->type)) { + addGroupInfoForSubquery(pSql, pNew, 0, tableIndex); + + // refactor as one method + SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); + assert(pNewQueryInfo != NULL); + + tscColumnBaseInfoUpdateTableIndex(&pNewQueryInfo->colList, 0); + tscColumnBaseInfoCopy(&pSupporter->colList, &pNewQueryInfo->colList, 0); + + tscSqlExprCopy(&pSupporter->exprsInfo, &pNewQueryInfo->exprsInfo, pSupporter->uid, false); + tscFieldInfoCopyAll(&pSupporter->fieldsInfo, &pNewQueryInfo->fieldsInfo); + + tscTagCondCopy(&pSupporter->tagCond, &pNewQueryInfo->tagCond); + + pNew->cmd.numOfCols = 0; + pNewQueryInfo->intervalTime = 0; + memset(&pNewQueryInfo->limit, 0, sizeof(SLimitVal)); + + // backup the data and clear it in the sqlcmd object + pSupporter->groupbyExpr = pNewQueryInfo->groupbyExpr; + memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SSqlGroupbyExpr)); + + // this data needs to be transfer to support struct + pNewQueryInfo->fieldsInfo.numOfOutputCols = 0; + pNewQueryInfo->exprsInfo.numOfExprs = 0; + + // set the ts,tags that involved in join, as the output column of intermediate result + tscClearSubqueryInfo(&pNew->cmd); + + SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = 1}; + SColumnIndex index = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; + + tscAddSpecialColumnForSelect(pNewQueryInfo, 0, TSDB_FUNC_TS_COMP, &index, &colSchema, TSDB_COL_NORMAL); + + // set the tags value for ts_comp function + SSqlExpr *pExpr = tscSqlExprGet(pNewQueryInfo, 0); + + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0); + int16_t tagColIndex = tscGetJoinTagColIndexByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->uid); + + pExpr->param->i64Key = tagColIndex; + pExpr->numOfParams = 1; + + // add the filter tag column + for (int32_t i = 0; i < pSupporter->colList.numOfCols; ++i) { + SColumnBase *pColBase = &pSupporter->colList.pColList[i]; + if (pColBase->numOfFilters > 0) { // copy to the pNew->cmd.colList if it is filtered. + tscColumnBaseCopy(&pNewQueryInfo->colList.pColList[pNewQueryInfo->colList.numOfCols], pColBase); + pNewQueryInfo->colList.numOfCols++; + } + } + + tscTrace("%p subquery:%p tableIndex:%d, vnodeIdx:%d, type:%d, transfer to ts_comp query to retrieve timestamps, " + "exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s", + pSql, pNew, tableIndex, pTableMetaInfo->vnodeIndex, pNewQueryInfo->type, + pNewQueryInfo->exprsInfo.numOfExprs, pNewQueryInfo->colList.numOfCols, + pNewQueryInfo->fieldsInfo.numOfOutputCols, pNewQueryInfo->pTableMetaInfo[0]->name); + tscPrintSelectClause(pNew, 0); + + tscTrace("%p subquery:%p tableIndex:%d, vnodeIdx:%d, type:%d, transfer to ts_comp query to retrieve timestamps, " + "exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s", + pSql, pNew, tableIndex, pTableMetaInfo->vnodeIndex, pNewQueryInfo->type, + pNewQueryInfo->exprsInfo.numOfExprs, pNewQueryInfo->colList.numOfCols, + pNewQueryInfo->fieldsInfo.numOfOutputCols, pNewQueryInfo->pTableMetaInfo[0]->name); + tscPrintSelectClause(pNew, 0); + } else { + SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); + pNewQueryInfo->type |= TSDB_QUERY_TYPE_SUBQUERY; + } + +#ifdef _DEBUG_VIEW + tscPrintSelectClause(pNew, 0); +#endif + + return tscProcessSql(pNew); +} + +// todo support async join query +int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) { + SSqlCmd* pCmd = &pSql->cmd; + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + + assert((pQueryInfo->type & TSDB_QUERY_TYPE_SUBQUERY) == 0); + + SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); + + pState->numOfTotal = pQueryInfo->numOfTables; + + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + SJoinSubquerySupporter *pSupporter = tscCreateJoinSupporter(pSql, pState, i); + + if (pSupporter == NULL) { // failed to create support struct, abort current query + tscError("%p tableIndex:%d, failed to allocate join support object, abort further query", pSql, i); + pState->numOfCompleted = pQueryInfo->numOfTables - i - 1; + pSql->res.code = TSDB_CODE_CLI_OUT_OF_MEMORY; + + return pSql->res.code; + } + + int32_t code = tscLaunchJoinSubquery(pSql, i, pSupporter); + if (code != TSDB_CODE_SUCCESS) { // failed to create subquery object, quit query + tscDestroyJoinSupporter(pSupporter); + pSql->res.code = TSDB_CODE_CLI_OUT_OF_MEMORY; + + break; + } + } + + tsem_post(&pSql->emptyRspSem); + tsem_wait(&pSql->rspSem); + + tsem_post(&pSql->emptyRspSem); + + if (pSql->numOfSubs <= 0) { + pSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; + } else { + pSql->cmd.command = TSDB_SQL_METRIC_JOIN_RETRIEVE; + } + + return TSDB_CODE_SUCCESS; +} + +static void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs, SSubqueryState* pState) { + assert(numOfSubs <= pSql->numOfSubs && numOfSubs >= 0 && pState != NULL); + + for(int32_t i = 0; i < numOfSubs; ++i) { + SSqlObj* pSub = pSql->pSubs[i]; + assert(pSub != NULL); + + SRetrieveSupport* pSupport = pSub->param; + + tfree(pSupport->localBuffer); + + pthread_mutex_unlock(&pSupport->queryMutex); + pthread_mutex_destroy(&pSupport->queryMutex); + + tfree(pSupport); + + tscFreeSqlObj(pSub); + } + + free(pState); +} + +int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { + SSqlRes *pRes = &pSql->res; + SSqlCmd *pCmd = &pSql->cmd; + + // pRes->code check only serves in launching metric sub-queries + if (pRes->code == TSDB_CODE_QUERY_CANCELLED) { + pCmd->command = TSDB_SQL_RETRIEVE_METRIC; // enable the abort of kill metric function. + return pRes->code; + } + + tExtMemBuffer ** pMemoryBuf = NULL; + tOrderDescriptor *pDesc = NULL; + SColumnModel * pModel = NULL; + + pRes->qhandle = 1; // hack the qhandle check + + const uint32_t nBufferSize = (1 << 16); // 64KB + + SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + int32_t numOfSubQueries = pTableMetaInfo->pMetricMeta->numOfVnodes; + assert(numOfSubQueries > 0); + + int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, nBufferSize); + if (ret != 0) { + pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; + if (pSql->fp) { + tscQueueAsyncRes(pSql); + } + return pRes->code; + } + + pSql->pSubs = calloc(numOfSubQueries, POINTER_BYTES); + pSql->numOfSubs = numOfSubQueries; + + tscTrace("%p retrieved query data from %d vnode(s)", pSql, numOfSubQueries); + SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); + pState->numOfTotal = numOfSubQueries; + pRes->code = TSDB_CODE_SUCCESS; + + int32_t i = 0; + for (; i < numOfSubQueries; ++i) { + SRetrieveSupport *trs = (SRetrieveSupport *)calloc(1, sizeof(SRetrieveSupport)); + if (trs == NULL) { + tscError("%p failed to malloc buffer for SRetrieveSupport, orderOfSub:%d, reason:%s", pSql, i, strerror(errno)); + break; + } + + trs->pExtMemBuffer = pMemoryBuf; + trs->pOrderDescriptor = pDesc; + trs->pState = pState; + + trs->localBuffer = (tFilePage *)calloc(1, nBufferSize + sizeof(tFilePage)); + if (trs->localBuffer == NULL) { + tscError("%p failed to malloc buffer for local buffer, orderOfSub:%d, reason:%s", pSql, i, strerror(errno)); + tfree(trs); + break; + } + + trs->subqueryIndex = i; + trs->pParentSqlObj = pSql; + trs->pFinalColModel = pModel; + + pthread_mutexattr_t mutexattr = {0}; + pthread_mutexattr_settype(&mutexattr, PTHREAD_MUTEX_RECURSIVE_NP); + pthread_mutex_init(&trs->queryMutex, &mutexattr); + pthread_mutexattr_destroy(&mutexattr); + + SSqlObj *pNew = tscCreateSqlObjForSubquery(pSql, trs, NULL); + if (pNew == NULL) { + tscError("%p failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql, i, strerror(errno)); + tfree(trs->localBuffer); + tfree(trs); + break; + } + + // todo handle multi-vnode situation + if (pQueryInfo->tsBuf) { + SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); + pNewQueryInfo->tsBuf = tsBufClone(pQueryInfo->tsBuf); + } + + tscTrace("%p sub:%p create subquery success. orderOfSub:%d", pSql, pNew, trs->subqueryIndex); + } + + if (i < numOfSubQueries) { + tscError("%p failed to prepare subquery structure and launch subqueries", pSql); + pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; + + tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, numOfSubQueries); + doCleanupSubqueries(pSql, i, pState); + return pRes->code; // free all allocated resource + } + + if (pRes->code == TSDB_CODE_QUERY_CANCELLED) { + tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, numOfSubQueries); + doCleanupSubqueries(pSql, i, pState); + return pRes->code; + } + + for(int32_t j = 0; j < numOfSubQueries; ++j) { + SSqlObj* pSub = pSql->pSubs[j]; + SRetrieveSupport* pSupport = pSub->param; + + tscTrace("%p sub:%p launch subquery, orderOfSub:%d.", pSql, pSub, pSupport->subqueryIndex); + tscProcessSql(pSub); + } + + return TSDB_CODE_SUCCESS; +} + +static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) { + tscTrace("%p start to free subquery result", pSql); + + if (pSql->res.code == TSDB_CODE_SUCCESS) { + taos_free_result(pSql); + } + + tfree(trsupport->localBuffer); + + pthread_mutex_unlock(&trsupport->queryMutex); + pthread_mutex_destroy(&trsupport->queryMutex); + + tfree(trsupport); +} + +static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows); + +static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES *tres, int32_t errCode) { +// set no disk space error info +#ifdef WINDOWS + LPVOID lpMsgBuf; + FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, + GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), // Default language + (LPTSTR)&lpMsgBuf, 0, NULL); + tscError("sub:%p failed to flush data to disk:reason:%s", tres, lpMsgBuf); + LocalFree(lpMsgBuf); +#else + char buf[256] = {0}; + strerror_r(errno, buf, 256); + tscError("sub:%p failed to flush data to disk:reason:%s", tres, buf); +#endif + + trsupport->pState->code = -errCode; + trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; + + pthread_mutex_unlock(&trsupport->queryMutex); + + tscRetrieveFromVnodeCallBack(trsupport, tres, trsupport->pState->code); +} + +static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows) { + SSqlObj *pPObj = trsupport->pParentSqlObj; + int32_t subqueryIndex = trsupport->subqueryIndex; + + assert(pSql != NULL); + SSubqueryState* pState = trsupport->pState; + assert(pState->numOfCompleted < pState->numOfTotal && pState->numOfCompleted >= 0 && + pPObj->numOfSubs == pState->numOfTotal); + + /* retrieved in subquery failed. OR query cancelled in retrieve phase. */ + if (pState->code == TSDB_CODE_SUCCESS && pPObj->res.code != TSDB_CODE_SUCCESS) { + pState->code = -(int)pPObj->res.code; + + /* + * kill current sub-query connection, which may retrieve data from vnodes; + * Here we get: pPObj->res.code == TSDB_CODE_QUERY_CANCELLED + */ + pSql->res.numOfRows = 0; + trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; // disable retry efforts + tscTrace("%p query is cancelled, sub:%p, orderOfSub:%d abort retrieve, code:%d", trsupport->pParentSqlObj, pSql, + subqueryIndex, pState->code); + } + + if (numOfRows >= 0) { // current query is successful, but other sub query failed, still abort current query. + tscTrace("%p sub:%p retrieve numOfRows:%d,orderOfSub:%d", pPObj, pSql, numOfRows, subqueryIndex); + tscError("%p sub:%p abort further retrieval due to other queries failure,orderOfSub:%d,code:%d", pPObj, pSql, + subqueryIndex, pState->code); + } else { + if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY && pState->code == TSDB_CODE_SUCCESS) { + /* + * current query failed, and the retry count is less than the available + * count, retry query clear previous retrieved data, then launch a new sub query + */ + tExtMemBufferClear(trsupport->pExtMemBuffer[subqueryIndex]); + + // clear local saved number of results + trsupport->localBuffer->numOfElems = 0; + pthread_mutex_unlock(&trsupport->queryMutex); + + tscTrace("%p sub:%p retrieve failed, code:%d, orderOfSub:%d, retry:%d", trsupport->pParentSqlObj, pSql, numOfRows, + subqueryIndex, trsupport->numOfRetry); + + SSqlObj *pNew = tscCreateSqlObjForSubquery(trsupport->pParentSqlObj, trsupport, pSql); + if (pNew == NULL) { + tscError("%p sub:%p failed to create new subquery sqlobj due to out of memory, abort retry", + trsupport->pParentSqlObj, pSql); + + pState->code = TSDB_CODE_CLI_OUT_OF_MEMORY; + trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; + return; + } + + tscProcessSql(pNew); + return; + } else { // reach the maximum retry count, abort + atomic_val_compare_exchange_32(&pState->code, TSDB_CODE_SUCCESS, numOfRows); + tscError("%p sub:%p retrieve failed,code:%d,orderOfSub:%d failed.no more retry,set global code:%d", pPObj, pSql, + numOfRows, subqueryIndex, pState->code); + } + } + + int32_t numOfTotal = pState->numOfTotal; + + int32_t finished = atomic_add_fetch_32(&pState->numOfCompleted, 1); + if (finished < numOfTotal) { + tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex, finished); + return tscFreeSubSqlObj(trsupport, pSql); + } + + // all subqueries are failed + tscError("%p retrieve from %d vnode(s) completed,code:%d.FAILED.", pPObj, pState->numOfTotal, pState->code); + pPObj->res.code = -(pState->code); + + // release allocated resource + tscLocalReducerEnvDestroy(trsupport->pExtMemBuffer, trsupport->pOrderDescriptor, trsupport->pFinalColModel, + pState->numOfTotal); + + tfree(trsupport->pState); + tscFreeSubSqlObj(trsupport, pSql); + + // in case of second stage join subquery, invoke its callback function instead of regular QueueAsyncRes + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pPObj->cmd, 0); + + if ((pQueryInfo->type & TSDB_QUERY_TYPE_JOIN_SEC_STAGE) == TSDB_QUERY_TYPE_JOIN_SEC_STAGE) { + (*pPObj->fp)(pPObj->param, pPObj, pPObj->res.code); + } else { // regular super table query + if (pPObj->res.code != TSDB_CODE_SUCCESS) { + tscQueueAsyncRes(pPObj); + } + } +} + +static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { + SRetrieveSupport *trsupport = (SRetrieveSupport *)param; + int32_t idx = trsupport->subqueryIndex; + SSqlObj * pPObj = trsupport->pParentSqlObj; + tOrderDescriptor *pDesc = trsupport->pOrderDescriptor; + + SSqlObj *pSql = (SSqlObj *)tres; + if (pSql == NULL) { // sql object has been released in error process, return immediately + tscTrace("%p subquery has been released, idx:%d, abort", pPObj, idx); + return; + } + + SSubqueryState* pState = trsupport->pState; + assert(pState->numOfCompleted < pState->numOfTotal && pState->numOfCompleted >= 0 && + pPObj->numOfSubs == pState->numOfTotal); + + // query process and cancel query process may execute at the same time + pthread_mutex_lock(&trsupport->queryMutex); + + if (numOfRows < 0 || pState->code < 0 || pPObj->res.code != TSDB_CODE_SUCCESS) { + return tscHandleSubRetrievalError(trsupport, pSql, numOfRows); + } + + SSqlRes * pRes = &pSql->res; + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); + + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + SVnodeSidList *vnodeInfo = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx); + SVnodeDesc * pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index]; + + if (numOfRows > 0) { + assert(pRes->numOfRows == numOfRows); + int64_t num = atomic_add_fetch_64(&pState->numOfRetrievedRows, numOfRows); + + tscTrace("%p sub:%p retrieve numOfRows:%d totalNumOfRows:%d from ip:%u,vid:%d,orderOfSub:%d", pPObj, pSql, + pRes->numOfRows, pState->numOfRetrievedRows, pSvd->ip, pSvd->vnode, idx); + + if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0)) { + tscError("%p sub:%p num of OrderedRes is too many, max allowed:%" PRId64 " , current:%" PRId64, + pPObj, pSql, tsMaxNumOfOrderedResults, num); + tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_SORTED_RES_TOO_MANY); + return; + } + + +#ifdef _DEBUG_VIEW + printf("received data from vnode: %d rows\n", pRes->numOfRows); + SSrcColumnInfo colInfo[256] = {0}; + + tscGetSrcColumnInfo(colInfo, pQueryInfo); + tColModelDisplayEx(pDesc->pColumnModel, pRes->data, pRes->numOfRows, pRes->numOfRows, colInfo); +#endif + if (tsTotalTmpDirGB != 0 && tsAvailTmpDirGB < tsMinimalTmpDirGB) { + tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pPObj, pSql, + tsAvailTmpDirGB, tsMinimalTmpDirGB); + tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE); + return; + } + + int32_t ret = saveToBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, pRes->data, + pRes->numOfRows, pQueryInfo->groupbyExpr.orderType); + if (ret < 0) { + // set no disk space error info, and abort retry + tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE); + } else { + pthread_mutex_unlock(&trsupport->queryMutex); + taos_fetch_rows_a(tres, tscRetrieveFromVnodeCallBack, param); + } + + } else { // all data has been retrieved to client + /* data in from current vnode is stored in cache and disk */ + uint32_t numOfRowsFromVnode = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->numOfElems; + tscTrace("%p sub:%p all data retrieved from ip:%u,vid:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql, pSvd->ip, + pSvd->vnode, numOfRowsFromVnode, idx); + + tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity); + +#ifdef _DEBUG_VIEW + printf("%" PRIu64 " rows data flushed to disk:\n", trsupport->localBuffer->numOfElems); + SSrcColumnInfo colInfo[256] = {0}; + tscGetSrcColumnInfo(colInfo, pQueryInfo); + tColModelDisplayEx(pDesc->pColumnModel, trsupport->localBuffer->data, trsupport->localBuffer->numOfElems, + trsupport->localBuffer->numOfElems, colInfo); +#endif + + if (tsTotalTmpDirGB != 0 && tsAvailTmpDirGB < tsMinimalTmpDirGB) { + tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pPObj, pSql, + tsAvailTmpDirGB, tsMinimalTmpDirGB); + tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE); + return; + } + + // each result for a vnode is ordered as an independant list, + // then used as an input of loser tree for disk-based merge routine + int32_t ret = tscFlushTmpBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, + pQueryInfo->groupbyExpr.orderType); + if (ret != 0) { + /* set no disk space error info, and abort retry */ + return tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE); + } + + // keep this value local variable, since the pState variable may be released by other threads, if atomic_add opertion + // increases the finished value up to pState->numOfTotal value, which means all subqueries are completed. + // In this case, the comparsion between finished value and released pState->numOfTotal is not safe. + int32_t numOfTotal = pState->numOfTotal; + + int32_t finished = atomic_add_fetch_32(&pState->numOfCompleted, 1); + if (finished < numOfTotal) { + tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex, finished); + return tscFreeSubSqlObj(trsupport, pSql); + } + + // all sub-queries are returned, start to local merge process + pDesc->pColumnModel->capacity = trsupport->pExtMemBuffer[idx]->numOfElemsPerPage; + + tscTrace("%p retrieve from %d vnodes completed.final NumOfRows:%d,start to build loser tree", pPObj, + pState->numOfTotal, pState->numOfRetrievedRows); + + SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pPObj->cmd, 0); + tscClearInterpInfo(pPQueryInfo); + + tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfTotal, pDesc, trsupport->pFinalColModel, + &pPObj->cmd, &pPObj->res); + tscTrace("%p build loser tree completed", pPObj); + + pPObj->res.precision = pSql->res.precision; + pPObj->res.numOfRows = 0; + pPObj->res.row = 0; + + // only free once + tfree(trsupport->pState); + tscFreeSubSqlObj(trsupport, pSql); + + // set the command flag must be after the semaphore been correctly set. + pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC; + if (pPObj->res.code == TSDB_CODE_SUCCESS) { + (*pPObj->fp)(pPObj->param, pPObj, 0); + } else { + tscQueueAsyncRes(pPObj); + } + } +} + +static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj) { + const int32_t table_index = 0; + + SSqlObj *pNew = createSubqueryObj(pSql, table_index, tscRetrieveDataRes, trsupport, prevSqlObj); + if (pNew != NULL) { // the sub query of two-stage super table query + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); + pQueryInfo->type |= TSDB_QUERY_TYPE_STABLE_SUBQUERY; + + assert(pQueryInfo->numOfTables == 1 && pNew->cmd.numOfClause == 1); + + // launch subquery for each vnode, so the subquery index equals to the vnodeIndex. + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, table_index); + pTableMetaInfo->vnodeIndex = trsupport->subqueryIndex; + + pSql->pSubs[trsupport->subqueryIndex] = pNew; + } + + return pNew; +} + +void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { + SRetrieveSupport *trsupport = (SRetrieveSupport *)param; + + SSqlObj* pParentSql = trsupport->pParentSqlObj; + SSqlObj* pSql = (SSqlObj *)tres; + + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); + assert(pSql->cmd.numOfClause == 1 && pSql->cmd.pQueryInfo[0]->numOfTables == 1); + + int32_t idx = pTableMetaInfo->vnodeIndex; + + SVnodeSidList *vnodeInfo = NULL; + SVnodeDesc * pSvd = NULL; + if (pTableMetaInfo->pMetricMeta != NULL) { + vnodeInfo = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx); + pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index]; + } + + SSubqueryState* pState = trsupport->pState; + assert(pState->numOfCompleted < pState->numOfTotal && pState->numOfCompleted >= 0 && + pParentSql->numOfSubs == pState->numOfTotal); + + if (pParentSql->res.code != TSDB_CODE_SUCCESS || pState->code != TSDB_CODE_SUCCESS) { + // metric query is killed, Note: code must be less than 0 + trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; + if (pParentSql->res.code != TSDB_CODE_SUCCESS) { + code = -(int)(pParentSql->res.code); + } else { + code = pState->code; + } + tscTrace("%p query cancelled or failed, sub:%p, orderOfSub:%d abort, code:%d", pParentSql, pSql, + trsupport->subqueryIndex, code); + } + + /* + * if a query on a vnode is failed, all retrieve operations from vnode that occurs later + * than this one are actually not necessary, we simply call the tscRetrieveFromVnodeCallBack + * function to abort current and remain retrieve process. + * + * NOTE: threadsafe is required. + */ + if (code != TSDB_CODE_SUCCESS) { + if (trsupport->numOfRetry++ >= MAX_NUM_OF_SUBQUERY_RETRY) { + tscTrace("%p sub:%p reach the max retry count,set global code:%d", pParentSql, pSql, code); + atomic_val_compare_exchange_32(&pState->code, 0, code); + } else { // does not reach the maximum retry count, go on + tscTrace("%p sub:%p failed code:%d, retry:%d", pParentSql, pSql, code, trsupport->numOfRetry); + + SSqlObj *pNew = tscCreateSqlObjForSubquery(pParentSql, trsupport, pSql); + if (pNew == NULL) { + tscError("%p sub:%p failed to create new subquery due to out of memory, abort retry, vid:%d, orderOfSub:%d", + trsupport->pParentSqlObj, pSql, pSvd != NULL ? pSvd->vnode : -1, trsupport->subqueryIndex); + + pState->code = -TSDB_CODE_CLI_OUT_OF_MEMORY; + trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; + } else { + SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); + assert(pNewQueryInfo->pTableMetaInfo[0]->pTableMeta != NULL && pNewQueryInfo->pTableMetaInfo[0]->pMetricMeta != NULL); + tscProcessSql(pNew); + return; + } + } + } + + if (pState->code != TSDB_CODE_SUCCESS) { // failed, abort + if (vnodeInfo != NULL) { + tscTrace("%p sub:%p query failed,ip:%u,vid:%d,orderOfSub:%d,global code:%d", pParentSql, pSql, + vnodeInfo->vpeerDesc[vnodeInfo->index].ip, vnodeInfo->vpeerDesc[vnodeInfo->index].vnode, + trsupport->subqueryIndex, pState->code); + } else { + tscTrace("%p sub:%p query failed,orderOfSub:%d,global code:%d", pParentSql, pSql, + trsupport->subqueryIndex, pState->code); + } + + tscRetrieveFromVnodeCallBack(param, tres, pState->code); + } else { // success, proceed to retrieve data from dnode + if (vnodeInfo != NULL) { + tscTrace("%p sub:%p query complete,ip:%u,vid:%d,orderOfSub:%d,retrieve data", trsupport->pParentSqlObj, pSql, + vnodeInfo->vpeerDesc[vnodeInfo->index].ip, vnodeInfo->vpeerDesc[vnodeInfo->index].vnode, + trsupport->subqueryIndex); + } else { + tscTrace("%p sub:%p query complete, orderOfSub:%d,retrieve data", trsupport->pParentSqlObj, pSql, + trsupport->subqueryIndex); + } + + taos_fetch_rows_a(tres, tscRetrieveFromVnodeCallBack, param); + } +} + +static void multiVnodeInsertMerge(void* param, TAOS_RES* tres, int numOfRows) { + SInsertSupporter *pSupporter = (SInsertSupporter *)param; + SSqlObj* pParentObj = pSupporter->pSql; + SSqlCmd* pParentCmd = &pParentObj->cmd; + + SSubqueryState* pState = pSupporter->pState; + int32_t total = pState->numOfTotal; + + // increase the total inserted rows + if (numOfRows > 0) { + pParentObj->res.numOfRows += numOfRows; + } + + int32_t completed = atomic_add_fetch_32(&pState->numOfCompleted, 1); + if (completed < total) { + return; + } + + tscTrace("%p Async insertion completed, total inserted:%d", pParentObj, pParentObj->res.numOfRows); + + // release data block data + pParentCmd->pDataBlocks = tscDestroyBlockArrayList(pParentCmd->pDataBlocks); + + // restore user defined fp + pParentObj->fp = pParentObj->fetchFp; + + // all data has been sent to vnode, call user function + (*pParentObj->fp)(pParentObj->param, tres, numOfRows); +} + +int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { + SSqlRes *pRes = &pSql->res; + SSqlCmd *pCmd = &pSql->cmd; + + pRes->qhandle = 1; // hack the qhandle check + SDataBlockList *pDataBlocks = pCmd->pDataBlocks; + + pSql->pSubs = calloc(pDataBlocks->nSize, POINTER_BYTES); + pSql->numOfSubs = pDataBlocks->nSize; + assert(pDataBlocks->nSize > 0); + + tscTrace("%p submit data to %d vnode(s)", pSql, pDataBlocks->nSize); + SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); + pState->numOfTotal = pSql->numOfSubs; + + pRes->code = TSDB_CODE_SUCCESS; + + int32_t i = 0; + for (; i < pSql->numOfSubs; ++i) { + SInsertSupporter* pSupporter = calloc(1, sizeof(SInsertSupporter)); + pSupporter->pSql = pSql; + pSupporter->pState = pState; + + SSqlObj *pNew = createSubqueryObj(pSql, 0, multiVnodeInsertMerge, pSupporter, NULL); + if (pNew == NULL) { + tscError("%p failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql, i, strerror(errno)); + break; + } + + pSql->pSubs[i] = pNew; + tscTrace("%p sub:%p create subObj success. orderOfSub:%d", pSql, pNew, i); + } + + if (i < pSql->numOfSubs) { + tscError("%p failed to prepare subObj structure and launch sub-insertion", pSql); + pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; + return pRes->code; // free all allocated resource + } + + for (int32_t j = 0; j < pSql->numOfSubs; ++j) { + SSqlObj *pSub = pSql->pSubs[j]; + pSub->cmd.command = TSDB_SQL_INSERT; + int32_t code = tscCopyDataBlockToPayload(pSub, pDataBlocks->pData[j]); + + if (code != TSDB_CODE_SUCCESS) { + tscTrace("%p prepare submit data block failed in async insertion, vnodeIdx:%d, total:%d, code:%d", pSql, j, + pDataBlocks->nSize, code); + } + + tscTrace("%p sub:%p launch sub insert, orderOfSub:%d", pSql, pSub, j); + tscProcessSql(pSub); + } + + return TSDB_CODE_SUCCESS; +} diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 426570134396956d06122bfd593226cac82a224f..faf5dd94b8aeef33b86af5856ad0198e2050f571 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -159,7 +159,7 @@ void taos_init_imp() { tscMgmtIpList.ip[1] = inet_addr(tsSecondIp); } - tscInitMsgs(); + tscInitMsgsFp(); slaveIndex = rand(); int queueSize = tsMaxVnodeConnections + tsMaxMeterConnections + tsMaxMgmtConnections + tsMaxMgmtConnections; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index c970b1db00fc8432a412b704c708d141ed46ebd9..3fbbc7e183e7529921eca616392ca7250d7264a5 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -21,9 +21,9 @@ #include "tcache.h" #include "tkey.h" #include "tmd5.h" -#include "tscJoinProcess.h" #include "tscProfile.h" #include "tscSecondaryMerge.h" +#include "tscSubquery.h" #include "tschemautil.h" #include "tsclient.h" #include "ttimer.h" @@ -40,12 +40,12 @@ */ void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* str, uint64_t uid) { int32_t index = -1; - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoByUid(pQueryInfo, uid, &index); + STableMetaInfo* pTableMetaInfo = tscGetMeterMetaInfoByUid(pQueryInfo, uid, &index); int32_t len = 0; char tagIdBuf[128] = {0}; - for (int32_t i = 0; i < pMeterMetaInfo->numOfTags; ++i) { - len += sprintf(&tagIdBuf[len], "%d,", pMeterMetaInfo->tagColumnIndex[i]); + for (int32_t i = 0; i < pTableMetaInfo->numOfTags; ++i) { + len += sprintf(&tagIdBuf[len], "%d,", pTableMetaInfo->tagColumnIndex[i]); } STagCond* pTagCond = &pQueryInfo->tagCond; @@ -64,7 +64,7 @@ void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* str, uint64_t uid) { size_t tbnameCondLen = pTagCond->tbnameCond.cond != NULL ? strlen(pTagCond->tbnameCond.cond) : 0; size_t redundantLen = 20; - size_t bufSize = strlen(pMeterMetaInfo->name) + tbnameCondLen + strlen(join) + strlen(tagIdBuf); + size_t bufSize = strlen(pTableMetaInfo->name) + tbnameCondLen + strlen(join) + strlen(tagIdBuf); if (cond != NULL && cond->cond != NULL) { bufSize += strlen(cond->cond); } @@ -72,7 +72,7 @@ void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* str, uint64_t uid) { bufSize = (size_t)((bufSize + redundantLen) * 1.5); char* tmp = calloc(1, bufSize); - int32_t keyLen = snprintf(tmp, bufSize, "%s,%s,%s,%d,%s,[%s],%d", pMeterMetaInfo->name, + int32_t keyLen = snprintf(tmp, bufSize, "%s,%s,%s,%d,%s,[%s],%d", pTableMetaInfo->name, ((cond != NULL && cond->cond != NULL) ? cond->cond : NULL), (tbnameCondLen > 0 ? pTagCond->tbnameCond.cond : NULL), pTagCond->relType, join, tagIdBuf, pQueryInfo->groupbyExpr.orderType); @@ -209,22 +209,22 @@ STableSidExtInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx) { return (STableSidExtInfo*)(pSidList->pSidExtInfoList[idx] + (char*)pSidList); } -bool tscIsTwoStageMergeMetricQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) { +bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) { if (pQueryInfo == NULL) { return false; } - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, tableIndex); - if (pMeterMetaInfo == NULL) { + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); + if (pTableMetaInfo == NULL) { return false; } // for select query super table, the metricmeta can not be null in any cases. - if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) { - assert(pMeterMetaInfo->pMetricMeta != NULL); + if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + assert(pTableMetaInfo->pMetricMeta != NULL); } - if (pMeterMetaInfo->pMetricMeta == NULL) { + if (pTableMetaInfo->pMetricMeta == NULL) { return false; } @@ -239,21 +239,21 @@ bool tscIsTwoStageMergeMetricQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) { if (((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_SUBQUERY) != TSDB_QUERY_TYPE_STABLE_SUBQUERY) && pQueryInfo->command == TSDB_SQL_SELECT) { - return UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo); + return UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo); } return false; } bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) { - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, tableIndex); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); /* * In following cases, return false for non ordered project query on super table * 1. failed to get metermeta from server; 2. not a super table; 3. limitation is 0; * 4. show queries, instead of a select query */ - if (pMeterMetaInfo == NULL || !UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo) || + if (pTableMetaInfo == NULL || !UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo) || pQueryInfo->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pQueryInfo->exprsInfo.numOfExprs == 0) { return false; } @@ -505,7 +505,7 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { tfree(pDataBlock->params); // free the refcount for metermeta - taosCacheRelease(tscCacheHandle, (void**)&(pDataBlock->pMeterMeta), false); + taosCacheRelease(tscCacheHandle, (void**)&(pDataBlock->pTableMeta), false); tfree(pDataBlock); } @@ -579,21 +579,21 @@ void* tscDestroyBlockArrayList(SDataBlockList* pList) { int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { SSqlCmd* pCmd = &pSql->cmd; - assert(pDataBlock->pMeterMeta != NULL); + assert(pDataBlock->pTableMeta != NULL); pCmd->numOfTablesInSubmit = pDataBlock->numOfTables; assert(pCmd->numOfClause == 1); - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); // set the correct metermeta object, the metermeta has been locked in pDataBlocks, so it must be in the cache - if (pMeterMetaInfo->pMeterMeta != pDataBlock->pMeterMeta) { - strcpy(pMeterMetaInfo->name, pDataBlock->tableId); - taosCacheRelease(tscCacheHandle, (void**)&(pMeterMetaInfo->pMeterMeta), false); + if (pTableMetaInfo->pTableMeta != pDataBlock->pTableMeta) { + strcpy(pTableMetaInfo->name, pDataBlock->tableId); + taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), false); - pMeterMetaInfo->pMeterMeta = taosCacheTransfer(tscCacheHandle, (void**)&pDataBlock->pMeterMeta); + pTableMetaInfo->pTableMeta = taosCacheTransfer(tscCacheHandle, (void**)&pDataBlock->pTableMeta); } else { - assert(strncmp(pMeterMetaInfo->name, pDataBlock->tableId, tListLen(pDataBlock->tableId)) == 0); + assert(strncmp(pTableMetaInfo->name, pDataBlock->tableId, tListLen(pDataBlock->tableId)) == 0); } /* @@ -637,7 +637,7 @@ void tscFreeUnusedDataBlocks(SDataBlockList* pList) { * @return */ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, const char* name, - STableMeta* pMeterMeta, STableDataBlocks** dataBlocks) { + STableMeta* pTableMeta, STableDataBlocks** dataBlocks) { STableDataBlocks* dataBuf = (STableDataBlocks*)calloc(1, sizeof(STableDataBlocks)); if (dataBuf == NULL) { tscError("failed to allocated memory, reason:%s", strerror(errno)); @@ -665,15 +665,15 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff * due to operation such as drop database. So here we add the reference count directly instead of invoke * taosGetDataFromCache, which may return NULL value. */ - dataBuf->pMeterMeta = taosCacheAcquireByData(tscCacheHandle, pMeterMeta); - assert(initialSize > 0 && pMeterMeta != NULL && dataBuf->pMeterMeta != NULL); + dataBuf->pTableMeta = taosCacheAcquireByData(tscCacheHandle, pTableMeta); + assert(initialSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); *dataBlocks = dataBuf; return TSDB_CODE_SUCCESS; } int32_t tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList, int64_t id, int32_t size, - int32_t startOffset, int32_t rowSize, const char* tableId, STableMeta* pMeterMeta, + int32_t startOffset, int32_t rowSize, const char* tableId, STableMeta* pTableMeta, STableDataBlocks** dataBlocks) { *dataBlocks = NULL; @@ -683,7 +683,7 @@ int32_t tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList, } if (*dataBlocks == NULL) { - int32_t ret = tscCreateDataBlock((size_t)size, rowSize, startOffset, tableId, pMeterMeta, dataBlocks); + int32_t ret = tscCreateDataBlock((size_t)size, rowSize, startOffset, tableId, pTableMeta, dataBlocks); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -707,7 +707,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi STableDataBlocks* dataBuf = NULL; int32_t ret = tscGetDataBlockFromList(pVnodeDataBlockHashList, pVnodeDataBlockList, pOneTableBlock->vgid, TSDB_PAYLOAD_SIZE, - tsInsertHeadSize, 0, pOneTableBlock->tableId, pOneTableBlock->pMeterMeta, &dataBuf); + tsInsertHeadSize, 0, pOneTableBlock->tableId, pOneTableBlock->pTableMeta, &dataBuf); if (ret != TSDB_CODE_SUCCESS) { tscError("%p failed to prepare the data block buffer for merging table data, code:%d", pSql, ret); taosHashCleanup(pVnodeDataBlockHashList); @@ -1061,8 +1061,8 @@ void tscClearFieldInfo(SFieldInfo* pFieldInfo) { for(int32_t i = 0; i < pFieldInfo->numOfOutputCols; ++i) { if (pFieldInfo->pExpr[i] != NULL) { - tSQLBinaryExprDestroy(&pFieldInfo->pExpr[i]->pBinExprInfo.pBinExpr, NULL); - tfree(pFieldInfo->pExpr[i]->pBinExprInfo.pReqColumns); + tSQLBinaryExprDestroy(&pFieldInfo->pExpr[i]->binExprInfo.pBinExpr, NULL); + tfree(pFieldInfo->pExpr[i]->binExprInfo.pReqColumns); tfree(pFieldInfo->pExpr[i]); } } @@ -1116,7 +1116,7 @@ SSqlExpr* tscSqlExprInsertEmpty(SQueryInfo* pQueryInfo, int32_t index, int16_t f SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type, int16_t size, int16_t interSize) { - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, pColIndex->tableIndex); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pColIndex->tableIndex); SSqlExprInfo* pExprInfo = &pQueryInfo->exprsInfo; @@ -1127,13 +1127,13 @@ SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functi pExprInfo->pExprs[index] = pExpr; pExpr->functionId = functionId; - int16_t numOfCols = pMeterMetaInfo->pMeterMeta->numOfColumns; + int16_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); // set the correct column index if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { pExpr->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX; } else { - SSchema* pSchema = tsGetColumnSchema(pMeterMetaInfo->pMeterMeta, pColIndex->columnIndex); + SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pColIndex->columnIndex); pExpr->colInfo.colId = pSchema->colId; } @@ -1153,7 +1153,7 @@ SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functi pExpr->resType = type; pExpr->resBytes = size; pExpr->interResBytes = interSize; - pExpr->uid = pMeterMetaInfo->pMeterMeta->uid; + pExpr->uid = pTableMetaInfo->pTableMeta->uid; pExprInfo->numOfExprs++; return pExpr; @@ -1161,7 +1161,7 @@ SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functi SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type, int16_t size) { - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); SSqlExprInfo* pExprInfo = &pQueryInfo->exprsInfo; if (index > pExprInfo->numOfExprs) { return NULL; @@ -1172,7 +1172,7 @@ SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functi pExpr->functionId = functionId; pExpr->colInfo.colIdx = srcColumnIndex; - pExpr->colInfo.colId = tsGetColumnSchema(pMeterMetaInfo->pMeterMeta, srcColumnIndex)->colId; + pExpr->colInfo.colId = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, srcColumnIndex)->colId; pExpr->resType = type; pExpr->resBytes = size; @@ -1573,17 +1573,19 @@ void tscIncStreamExecutionCount(void* pStream) { ps->num += 1; } -bool tscValidateColumnId(SMeterMetaInfo* pMeterMetaInfo, int32_t colId) { - if (pMeterMetaInfo->pMeterMeta == NULL) { +bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId) { + if (pTableMetaInfo->pTableMeta == NULL) { return false; } - if (colId == -1 && UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) { + if (colId == -1 && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { return true; } - SSchema* pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); - int32_t numOfTotal = pMeterMetaInfo->pMeterMeta->numOfTags + pMeterMetaInfo->pMeterMeta->numOfColumns; + SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); + + int32_t numOfTotal = tinfo.numOfTags + tinfo.numOfColumns; for (int32_t i = 0; i < numOfTotal; ++i) { if (pSchema[i].colId == colId) { @@ -1627,16 +1629,16 @@ void tscTagCondRelease(STagCond* pCond) { } void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo) { - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - SSchema* pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); pColInfo[i].functionId = pExpr->functionId; if (TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { - SSchema* pTagSchema = tsGetTagSchema(pMeterMetaInfo->pMeterMeta); - int16_t actualTagIndex = pMeterMetaInfo->tagColumnIndex[pExpr->colInfo.colIdx]; + SSchema* pTagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); + int16_t actualTagIndex = pTableMetaInfo->tagColumnIndex[pExpr->colInfo.colIdx]; pColInfo[i].type = (actualTagIndex != -1) ? pTagSchema[actualTagIndex].type : TSDB_DATA_TYPE_BINARY; } else { @@ -1723,10 +1725,10 @@ bool tscShouldFreeAsyncSqlObj(SSqlObj* pSql) { SDataBlockList* pDataBlocks = pCmd->pDataBlocks; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); assert(pQueryInfo->numOfTables == 1 || pQueryInfo->numOfTables == 2); - if (pDataBlocks == NULL || pMeterMetaInfo->vnodeIndex >= pDataBlocks->nSize) { + if (pDataBlocks == NULL || pTableMetaInfo->vnodeIndex >= pDataBlocks->nSize) { tscTrace("%p object should be release since all data blocks have been submit", pSql); return true; } else { @@ -1745,7 +1747,7 @@ bool tscShouldFreeAsyncSqlObj(SSqlObj* pSql) { * @param tableIndex denote the table index for join query, where more than one table exists * @return */ -SMeterMetaInfo* tscGetMeterMetaInfo(SSqlCmd* pCmd, int32_t clauseIndex, int32_t tableIndex) { +STableMetaInfo* tscGetTableMetaInfoFromCmd(SSqlCmd* pCmd, int32_t clauseIndex, int32_t tableIndex) { if (pCmd == NULL || pCmd->numOfClause == 0) { return NULL; } @@ -1753,20 +1755,20 @@ SMeterMetaInfo* tscGetMeterMetaInfo(SSqlCmd* pCmd, int32_t clauseIndex, int32_t assert(clauseIndex >= 0 && clauseIndex < pCmd->numOfClause); SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); - return tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, tableIndex); + return tscGetMetaInfo(pQueryInfo, tableIndex); } -SMeterMetaInfo* tscGetMeterMetaInfoFromQueryInfo(SQueryInfo* pQueryInfo, int32_t tableIndex) { +STableMetaInfo* tscGetMetaInfo(SQueryInfo* pQueryInfo, int32_t tableIndex) { assert(pQueryInfo != NULL); - if (pQueryInfo->pMeterInfo == NULL) { + if (pQueryInfo->pTableMetaInfo == NULL) { assert(pQueryInfo->numOfTables == 0); return NULL; } - assert(tableIndex >= 0 && tableIndex <= pQueryInfo->numOfTables && pQueryInfo->pMeterInfo != NULL); + assert(tableIndex >= 0 && tableIndex <= pQueryInfo->numOfTables && pQueryInfo->pTableMetaInfo != NULL); - return pQueryInfo->pMeterInfo[tableIndex]; + return pQueryInfo->pTableMetaInfo[tableIndex]; } SQueryInfo* tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex) { @@ -1795,11 +1797,11 @@ int32_t tscGetQueryInfoDetailSafely(SSqlCmd* pCmd, int32_t subClauseIndex, SQuer return TSDB_CODE_SUCCESS; } -SMeterMetaInfo* tscGetMeterMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, int32_t* index) { +STableMetaInfo* tscGetMeterMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, int32_t* index) { int32_t k = -1; for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - if (pQueryInfo->pMeterInfo[i]->pMeterMeta->uid == uid) { + if (pQueryInfo->pTableMetaInfo[i]->pTableMeta->uid == uid) { k = i; break; } @@ -1810,7 +1812,7 @@ SMeterMetaInfo* tscGetMeterMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, i } assert(k != -1); - return tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, k); + return tscGetMetaInfo(pQueryInfo, k); } int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { @@ -1872,37 +1874,37 @@ void tscFreeSubqueryInfo(SSqlCmd* pCmd) { tfree(pCmd->pQueryInfo); } -SMeterMetaInfo* tscAddMeterMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pMeterMeta, +STableMetaInfo* tscAddMeterMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, SSuperTableMeta* pMetricMeta, int16_t numOfTags, int16_t* tags) { - void* pAlloc = realloc(pQueryInfo->pMeterInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES); + void* pAlloc = realloc(pQueryInfo->pTableMetaInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES); if (pAlloc == NULL) { return NULL; } - pQueryInfo->pMeterInfo = pAlloc; - pQueryInfo->pMeterInfo[pQueryInfo->numOfTables] = calloc(1, sizeof(SMeterMetaInfo)); + pQueryInfo->pTableMetaInfo = pAlloc; + pQueryInfo->pTableMetaInfo[pQueryInfo->numOfTables] = calloc(1, sizeof(STableMetaInfo)); - SMeterMetaInfo* pMeterMetaInfo = pQueryInfo->pMeterInfo[pQueryInfo->numOfTables]; - assert(pMeterMetaInfo != NULL); + STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[pQueryInfo->numOfTables]; + assert(pTableMetaInfo != NULL); if (name != NULL) { assert(strlen(name) <= TSDB_TABLE_ID_LEN); - strcpy(pMeterMetaInfo->name, name); + strcpy(pTableMetaInfo->name, name); } - pMeterMetaInfo->pMeterMeta = pMeterMeta; - pMeterMetaInfo->pMetricMeta = pMetricMeta; - pMeterMetaInfo->numOfTags = numOfTags; + pTableMetaInfo->pTableMeta = pTableMeta; + pTableMetaInfo->pMetricMeta = pMetricMeta; + pTableMetaInfo->numOfTags = numOfTags; if (tags != NULL) { - memcpy(pMeterMetaInfo->tagColumnIndex, tags, sizeof(pMeterMetaInfo->tagColumnIndex[0]) * numOfTags); + memcpy(pTableMetaInfo->tagColumnIndex, tags, sizeof(pTableMetaInfo->tagColumnIndex[0]) * numOfTags); } pQueryInfo->numOfTables += 1; - return pMeterMetaInfo; + return pTableMetaInfo; } -SMeterMetaInfo* tscAddEmptyMeterMetaInfo(SQueryInfo* pQueryInfo) { +STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo* pQueryInfo) { return tscAddMeterMetaInfo(pQueryInfo, NULL, NULL, NULL, 0, NULL); } @@ -1911,14 +1913,14 @@ void doRemoveMeterMetaInfo(SQueryInfo* pQueryInfo, int32_t index, bool removeFro return; } - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, index); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index); - tscClearMeterMetaInfo(pMeterMetaInfo, removeFromCache); - free(pMeterMetaInfo); + tscClearMeterMetaInfo(pTableMetaInfo, removeFromCache); + free(pTableMetaInfo); int32_t after = pQueryInfo->numOfTables - index - 1; if (after > 0) { - memmove(&pQueryInfo->pMeterInfo[index], &pQueryInfo->pMeterInfo[index + 1], after * POINTER_BYTES); + memmove(&pQueryInfo->pTableMetaInfo[index], &pQueryInfo->pTableMetaInfo[index + 1], after * POINTER_BYTES); } pQueryInfo->numOfTables -= 1; @@ -1932,16 +1934,16 @@ void tscRemoveAllMeterMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool doRemoveMeterMetaInfo(pQueryInfo, --index, removeFromCache); } - tfree(pQueryInfo->pMeterInfo); + tfree(pQueryInfo->pTableMetaInfo); } -void tscClearMeterMetaInfo(SMeterMetaInfo* pMeterMetaInfo, bool removeFromCache) { - if (pMeterMetaInfo == NULL) { +void tscClearMeterMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) { + if (pTableMetaInfo == NULL) { return; } - taosCacheRelease(tscCacheHandle, (void**)&(pMeterMetaInfo->pMeterMeta), removeFromCache); - taosCacheRelease(tscCacheHandle, (void**)&(pMeterMetaInfo->pMetricMeta), removeFromCache); + taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache); + taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pMetricMeta), removeFromCache); } void tscResetForNextRetrieve(SSqlRes* pRes) { @@ -1955,11 +1957,11 @@ void tscResetForNextRetrieve(SSqlRes* pRes) { SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void* param, SSqlObj* pPrevSql) { SSqlCmd* pCmd = &pSql->cmd; - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, tableIndex); + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, tableIndex); SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); if (pNew == NULL) { - tscError("%p new subquery failed, tableIndex:%d, vnodeIndex:%d", pSql, tableIndex, pMeterMetaInfo->vnodeIndex); + tscError("%p new subquery failed, tableIndex:%d, vnodeIndex:%d", pSql, tableIndex, pTableMetaInfo->vnodeIndex); return NULL; } @@ -1968,7 +1970,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pNew->sqlstr = strdup(pSql->sqlstr); if (pNew->sqlstr == NULL) { - tscError("%p new subquery failed, tableIndex:%d, vnodeIndex:%d", pSql, tableIndex, pMeterMetaInfo->vnodeIndex); + tscError("%p new subquery failed, tableIndex:%d, vnodeIndex:%d", pSql, tableIndex, pTableMetaInfo->vnodeIndex); free(pNew); return NULL; @@ -1997,7 +1999,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void memset(&pNewQueryInfo->colList, 0, sizeof(pNewQueryInfo->colList)); memset(&pNewQueryInfo->fieldsInfo, 0, sizeof(SFieldInfo)); - pNewQueryInfo->pMeterInfo = NULL; + pNewQueryInfo->pTableMetaInfo = NULL; pNewQueryInfo->defaultVal = NULL; pNewQueryInfo->numOfTables = 0; pNewQueryInfo->tsBuf = NULL; @@ -2010,7 +2012,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void } if (tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE) != TSDB_CODE_SUCCESS) { - tscError("%p new subquery failed, tableIndex:%d, vnodeIndex:%d", pSql, tableIndex, pMeterMetaInfo->vnodeIndex); + tscError("%p new subquery failed, tableIndex:%d, vnodeIndex:%d", pSql, tableIndex, pTableMetaInfo->vnodeIndex); tscFreeSqlObj(pNew); return NULL; } @@ -2025,7 +2027,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pNewQueryInfo->type |= TSDB_QUERY_TYPE_SUBQUERY; // it must be the subquery } - uint64_t uid = pMeterMetaInfo->pMeterMeta->uid; + uint64_t uid = pTableMetaInfo->pTableMeta->uid; tscSqlExprCopy(&pNewQueryInfo->exprsInfo, &pQueryInfo->exprsInfo, uid, true); int32_t numOfOutputCols = pNewQueryInfo->exprsInfo.numOfExprs; @@ -2067,34 +2069,34 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void printf("the metricmeta key is:%s\n", key); #endif - char* name = pMeterMetaInfo->name; - SMeterMetaInfo* pFinalInfo = NULL; + char* name = pTableMetaInfo->name; + STableMetaInfo* pFinalInfo = NULL; if (pPrevSql == NULL) { - STableMeta* pMeterMeta = taosCacheAcquireByName(tscCacheHandle, name); + STableMeta* pTableMeta = taosCacheAcquireByName(tscCacheHandle, name); SSuperTableMeta* pMetricMeta = taosCacheAcquireByName(tscCacheHandle, key); - pFinalInfo = tscAddMeterMetaInfo(pNewQueryInfo, name, pMeterMeta, pMetricMeta, pMeterMetaInfo->numOfTags, - pMeterMetaInfo->tagColumnIndex); - } else { // transfer the ownership of pMeterMeta/pMetricMeta to the newly create sql object. - SMeterMetaInfo* pPrevInfo = tscGetMeterMetaInfo(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0); + pFinalInfo = tscAddMeterMetaInfo(pNewQueryInfo, name, pTableMeta, pMetricMeta, pTableMetaInfo->numOfTags, + pTableMetaInfo->tagColumnIndex); + } else { // transfer the ownership of pTableMeta/pMetricMeta to the newly create sql object. + STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0); - STableMeta* pPrevMeterMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pMeterMeta); + STableMeta* pPrevMeterMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pTableMeta); SSuperTableMeta* pPrevMetricMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pMetricMeta); - pFinalInfo = tscAddMeterMetaInfo(pNewQueryInfo, name, pPrevMeterMeta, pPrevMetricMeta, pMeterMetaInfo->numOfTags, - pMeterMetaInfo->tagColumnIndex); + pFinalInfo = tscAddMeterMetaInfo(pNewQueryInfo, name, pPrevMeterMeta, pPrevMetricMeta, pTableMetaInfo->numOfTags, + pTableMetaInfo->tagColumnIndex); } - assert(pFinalInfo->pMeterMeta != NULL && pNewQueryInfo->numOfTables == 1); - if (UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) { + assert(pFinalInfo->pTableMeta != NULL && pNewQueryInfo->numOfTables == 1); + if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { assert(pFinalInfo->pMetricMeta != NULL); } tscTrace( "%p new subquery: %p, tableIndex:%d, vnodeIdx:%d, type:%d, exprInfo:%d, colList:%d," "fieldInfo:%d, name:%s, qrang:%" PRId64 " - %" PRId64 " order:%d, limit:%" PRId64, - pSql, pNew, tableIndex, pMeterMetaInfo->vnodeIndex, pNewQueryInfo->type, pNewQueryInfo->exprsInfo.numOfExprs, + pSql, pNew, tableIndex, pTableMetaInfo->vnodeIndex, pNewQueryInfo->type, pNewQueryInfo->exprsInfo.numOfExprs, pNewQueryInfo->colList.numOfCols, pNewQueryInfo->fieldsInfo.numOfOutputCols, pFinalInfo->name, pNewQueryInfo->stime, pNewQueryInfo->etime, pNewQueryInfo->order.order, pNewQueryInfo->limit.limit); @@ -2187,14 +2189,14 @@ bool hasMoreVnodesToTry(SSqlObj* pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - if (!UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo) || (pMeterMetaInfo->pMetricMeta == NULL)) { + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo) || (pTableMetaInfo->pMetricMeta == NULL)) { return false; } - int32_t totalVnode = pMeterMetaInfo->pMetricMeta->numOfVnodes; + int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes; return pRes->numOfRows == 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && - (!tscHasReachLimitation(pQueryInfo, pRes)) && (pMeterMetaInfo->vnodeIndex < totalVnode - 1); + (!tscHasReachLimitation(pQueryInfo, pRes)) && (pTableMetaInfo->vnodeIndex < totalVnode - 1); } void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) { @@ -2209,12 +2211,12 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) { */ assert(pRes->numOfRows == 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && !tscHasReachLimitation(pQueryInfo, pRes)); - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); - int32_t totalVnode = pMeterMetaInfo->pMetricMeta->numOfVnodes; + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes; - while (++pMeterMetaInfo->vnodeIndex < totalVnode) { + while (++pTableMetaInfo->vnodeIndex < totalVnode) { tscTrace("%p current vnode:%d exhausted, try next:%d. total vnode:%d. current numOfRes:%d", pSql, - pMeterMetaInfo->vnodeIndex - 1, pMeterMetaInfo->vnodeIndex, totalVnode, pRes->numOfTotalInCurrentClause); + pTableMetaInfo->vnodeIndex - 1, pTableMetaInfo->vnodeIndex, totalVnode, pRes->numOfTotalInCurrentClause); /* * update the limit and offset value for the query on the next vnode, @@ -2233,7 +2235,7 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) { assert((pRes->offset >= 0 && pRes->numOfRows == 0) || (pRes->offset == 0 && pRes->numOfRows >= 0)); tscTrace("%p new query to next vnode, vnode index:%d, limit:%" PRId64 ", offset:%" PRId64 ", glimit:%" PRId64, pSql, - pMeterMetaInfo->vnodeIndex, pQueryInfo->limit.limit, pQueryInfo->limit.offset, pQueryInfo->clauseLimit); + pTableMetaInfo->vnodeIndex, pQueryInfo->limit.limit, pQueryInfo->limit.offset, pQueryInfo->clauseLimit); /* * For project query with super table join, the numOfSub is equalled to the number of all subqueries. @@ -2319,94 +2321,3 @@ void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)()) { tscProcessSql(pSql); } } - -typedef struct SinsertSupporter { - SSubqueryState* pState; - SSqlObj* pSql; -} SinsertSupporter; - -void multiVnodeInsertMerge(void* param, TAOS_RES* tres, int numOfRows) { - SinsertSupporter *pSupporter = (SinsertSupporter *)param; - SSqlObj* pParentObj = pSupporter->pSql; - SSqlCmd* pParentCmd = &pParentObj->cmd; - - SSubqueryState* pState = pSupporter->pState; - int32_t total = pState->numOfTotal; - - // increase the total inserted rows - if (numOfRows > 0) { - pParentObj->res.numOfRows += numOfRows; - } - - int32_t completed = atomic_add_fetch_32(&pState->numOfCompleted, 1); - if (completed < total) { - return; - } - - tscTrace("%p Async insertion completed, total inserted:%d", pParentObj, pParentObj->res.numOfRows); - - // release data block data - pParentCmd->pDataBlocks = tscDestroyBlockArrayList(pParentCmd->pDataBlocks); - - // restore user defined fp - pParentObj->fp = pParentObj->fetchFp; - - // all data has been sent to vnode, call user function - (*pParentObj->fp)(pParentObj->param, tres, numOfRows); -} - -int32_t launchMultivnodeInsert(SSqlObj *pSql) { - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - pRes->qhandle = 1; // hack the qhandle check - SDataBlockList *pDataBlocks = pCmd->pDataBlocks; - - pSql->pSubs = calloc(pDataBlocks->nSize, POINTER_BYTES); - pSql->numOfSubs = pDataBlocks->nSize; - assert(pDataBlocks->nSize > 0); - - tscTrace("%p submit data to %d vnode(s)", pSql, pDataBlocks->nSize); - SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); - pState->numOfTotal = pSql->numOfSubs; - - pRes->code = TSDB_CODE_SUCCESS; - - int32_t i = 0; - for (; i < pSql->numOfSubs; ++i) { - SinsertSupporter* pSupporter = calloc(1, sizeof(SinsertSupporter)); - pSupporter->pSql = pSql; - pSupporter->pState = pState; - - SSqlObj *pNew = createSubqueryObj(pSql, 0, multiVnodeInsertMerge, pSupporter, NULL); - if (pNew == NULL) { - tscError("%p failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql, i, strerror(errno)); - break; - } - - pSql->pSubs[i] = pNew; - tscTrace("%p sub:%p create subObj success. orderOfSub:%d", pSql, pNew, i); - } - - if (i < pSql->numOfSubs) { - tscError("%p failed to prepare subObj structure and launch sub-insertion", pSql); - pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; - return pRes->code; // free all allocated resource - } - - for (int32_t j = 0; j < pSql->numOfSubs; ++j) { - SSqlObj *pSub = pSql->pSubs[j]; - pSub->cmd.command = TSDB_SQL_INSERT; - int32_t code = tscCopyDataBlockToPayload(pSub, pDataBlocks->pData[j]); - - if (code != TSDB_CODE_SUCCESS) { - tscTrace("%p prepare submit data block failed in async insertion, vnodeIdx:%d, total:%d, code:%d", pSql, j, - pDataBlocks->nSize, code); - } - - tscTrace("%p sub:%p launch sub insert, orderOfSub:%d", pSql, pSub, j); - tscProcessSql(pSub); - } - - return TSDB_CODE_SUCCESS; -} diff --git a/src/common/inc/name.h b/src/common/inc/name.h new file mode 100644 index 0000000000000000000000000000000000000000..31a6d8fbeb15123aa88a6f68501715822019a632 --- /dev/null +++ b/src/common/inc/name.h @@ -0,0 +1,26 @@ +#ifndef TDENGINE_NAME_H +#define TDENGINE_NAME_H + +#include "os.h" +#include "taosmsg.h" + +typedef struct SDataStatis { + int16_t colId; + int64_t sum; + int64_t max; + int64_t min; + int16_t maxIndex; + int16_t minIndex; + int16_t numOfNull; +} SDataStatis; + +typedef struct SColumnInfoEx { + SColumnInfo info; + void* pData; // the corresponding block data in memory +} SColumnInfoEx; + +int32_t extractTableName(const char *tableId, char *name); + +char* extractDBName(const char *tableId, char *name); + +#endif // TDENGINE_NAME_H diff --git a/src/common/src/name.c b/src/common/src/name.c new file mode 100644 index 0000000000000000000000000000000000000000..cf1eaa2bf11ce6ac6707e26d124feeaacb19fb20 --- /dev/null +++ b/src/common/src/name.c @@ -0,0 +1,42 @@ +#include "os.h" +#include "tutil.h" + +#include "name.h" +#include "tstoken.h" +#include "ttokendef.h" + +// todo refactor +static FORCE_INLINE const char* skipSegments(const char* input, char delim, int32_t num) { + for (int32_t i = 0; i < num; ++i) { + while (*input != 0 && *input++ != delim) { + }; + } + return input; +} + +static FORCE_INLINE size_t copy(char* dst, const char* src, char delimiter) { + size_t len = 0; + while (*src != delimiter && *src != 0) { + *dst++ = *src++; + len++; + } + + return len; +} + +int32_t extractTableName(const char* tableId, char* name) { + size_t offset = strcspn(tableId, &TS_PATH_DELIMITER[0]); + offset = strcspn(&tableId[offset], &TS_PATH_DELIMITER[0]); + + return strncpy(name, &tableId[offset], TSDB_TABLE_NAME_LEN); + +// char* r = skipSegments(tableId, TS_PATH_DELIMITER[0], 2); +// return copy(name, r, TS_PATH_DELIMITER[0]); +} + +char* extractDBName(const char* tableId, char* name) { + size_t offset1 = strcspn(tableId, &TS_PATH_DELIMITER[0]); + size_t len = strcspn(&tableId[offset1 + 1], &TS_PATH_DELIMITER[0]); + + return strncpy(name, &tableId[offset1 + 1], len); +} diff --git a/src/inc/mnode.h b/src/inc/mnode.h index b7631674b3bed837a230ded456345f6b54251c6f..1452e1f53a73bc5e0b0dc7dff0706873a9b9ed19 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -98,7 +98,7 @@ typedef struct { } SVnodeGid; typedef struct { - char tableId[TSDB_TABLE_ID_LEN + 1]; + char tableId[TSDB_TABLE_ID_LEN]; int8_t type; int8_t dirty; uint64_t uid; @@ -142,7 +142,7 @@ typedef struct { } SChildTableObj; typedef struct { - char tableId[TSDB_TABLE_ID_LEN + 1]; + char tableId[TSDB_TABLE_ID_LEN]; int8_t type; int8_t dirty; uint64_t uid; diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 49ed45559f1e9ff0fb644b9425a6bb604fa580bb..570e2e1acd0391f2f59f56b9b68b52e38310da14 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -177,7 +177,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_MIN_COLUMNS 2 //PRIMARY COLUMN(timestamp) + other columns #define TSDB_DNODE_NAME_LEN 63 -#define TSDB_TABLE_NAME_LEN 64 +#define TSDB_TABLE_NAME_LEN 192 #define TSDB_DB_NAME_LEN 32 #define TSDB_COL_NAME_LEN 64 #define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 16 diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index c908d34136db2526d0f4fb73b9f547c6aacc4573..0a885e9c4be1d7782dfbb7f39c3de1e2e81f77f3 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -180,9 +180,9 @@ enum _mgmt_table { #define TSDB_VN_WRITE_ACCCESS ((char)0x2) #define TSDB_VN_ALL_ACCCESS (TSDB_VN_READ_ACCCESS | TSDB_VN_WRITE_ACCCESS) -#define TSDB_COL_NORMAL 0x0U -#define TSDB_COL_TAG 0x1U -#define TSDB_COL_JOIN 0x2U +#define TSDB_COL_NORMAL 0x0u +#define TSDB_COL_TAG 0x1u +#define TSDB_COL_JOIN 0x2u extern char *taosMsg[]; @@ -222,13 +222,12 @@ typedef struct { typedef struct SSchema { uint8_t type; - char name[TSDB_COL_NAME_LEN + 1]; + char name[TSDB_COL_NAME_LEN]; int16_t colId; int16_t bytes; } SSchema; typedef struct { - int32_t vgId; int32_t vnode; //the index of vnode uint32_t ip; } SVnodeDesc; @@ -255,14 +254,14 @@ typedef struct { uint64_t uid; uint64_t superTableUid; uint64_t createdTime; - char tableId[TSDB_TABLE_ID_LEN + 1]; - char superTableId[TSDB_TABLE_ID_LEN + 1]; + char tableId[TSDB_TABLE_ID_LEN]; + char superTableId[TSDB_TABLE_ID_LEN]; char data[]; } SMDCreateTableMsg; typedef struct { - char tableId[TSDB_TABLE_ID_LEN + 1]; - char db[TSDB_DB_NAME_LEN + 1]; + char tableId[TSDB_TABLE_ID_LEN]; + char db[TSDB_DB_NAME_LEN]; int8_t igExists; int16_t numOfTags; int16_t numOfColumns; @@ -273,13 +272,13 @@ typedef struct { } SCMCreateTableMsg; typedef struct { - char tableId[TSDB_TABLE_ID_LEN + 1]; + char tableId[TSDB_TABLE_ID_LEN]; int8_t igNotExists; } SCMDropTableMsg; typedef struct { - char tableId[TSDB_TABLE_ID_LEN + 1]; - char db[TSDB_DB_NAME_LEN + 1]; + char tableId[TSDB_TABLE_ID_LEN]; + char db[TSDB_DB_NAME_LEN]; int16_t type; /* operation type */ char tagVal[TSDB_MAX_BYTES_PER_ROW]; int8_t numOfCols; /* number of schema */ @@ -396,7 +395,7 @@ typedef struct SSqlBinaryExprInfo { typedef struct SSqlFunctionExpr { SSqlFuncExprMsg pBase; - SSqlBinaryExprInfo pBinExprInfo; + SSqlBinaryExprInfo binExprInfo; int16_t resBytes; int16_t resType; int16_t interResBytes; @@ -445,6 +444,11 @@ typedef struct STableSidExtInfo { char tags[]; } STableSidExtInfo; +typedef struct STimeWindow { + TSKEY skey; + TSKEY ekey; +} STimeWindow; + /* * the outputCols is equalled to or larger than numOfCols * e.g., select min(colName), max(colName), avg(colName) from table @@ -452,46 +456,45 @@ typedef struct STableSidExtInfo { */ typedef struct { int16_t vnode; - int32_t numOfSids; + int32_t numOfTables; uint64_t pSidExtInfo; // table id & tag info ptr, in windows pointer may uint64_t uid; - TSKEY skey; - TSKEY ekey; + STimeWindow window; int16_t order; int16_t orderColId; int16_t numOfCols; // the number of columns will be load from vnode - char intervalTimeUnit; // time interval type, for revisement of interval(1d) + char slidingTimeUnit; // time interval type, for revisement of interval(1d) - int64_t intervalTime; // time interval for aggregation, in million second + int64_t intervalTime; // time interval for aggregation, in million second int64_t slidingTime; // value for sliding window // tag schema, used to parse tag information in pSidExtInfo uint64_t pTagSchema; - int16_t numOfTagsCols; // required number of tags - int16_t tagLength; // tag length in current query + int16_t numOfTagsCols; // required number of tags + int16_t tagLength; // tag length in current query int16_t numOfGroupCols; // num of group by columns int16_t orderByIdx; int16_t orderType; // used in group by xx order by xxx uint64_t groupbyTagIds; - int64_t limit; - int64_t offset; + int64_t limit; + int64_t offset; - int16_t queryType; // denote another query process - int16_t numOfOutputCols; // final output columns numbers + int16_t queryType; // denote another query process + int16_t numOfOutputCols; // final output columns numbers int16_t interpoType; // interpolate type uint64_t defaultVal; // default value array list - int32_t colNameLen; - int64_t colNameList; + int32_t colNameLen; + int64_t colNameList; - int64_t pSqlFuncExprs; + int64_t pSqlFuncExprs; int32_t tsOffset; // offset value in current msg body, NOTE: ts list is compressed int32_t tsLen; // total length of ts comp block @@ -677,28 +680,27 @@ typedef struct { int32_t list[]; /* offset of SVnodeSidList, compared to the SSuperTableMeta struct */ } SSuperTableMeta; -typedef struct STableMeta { - char tableId[TSDB_TABLE_ID_LEN + 1]; // note: This field must be at the front +typedef struct STableMetaMsg { + char tableId[TSDB_TABLE_ID_LEN]; // note: This field must be at the front int32_t contLen; - uint8_t numOfTags : 6; - uint8_t precision : 2; - uint8_t tableType : 4; - uint8_t index : 4; // used locally + uint8_t numOfTags; + uint8_t precision; + uint8_t tableType; int16_t numOfColumns; - int16_t rowSize; // used locally, calculated in client int16_t sversion; + int8_t numOfVpeers; SVnodeDesc vpeerDesc[TSDB_VNODES_SUPPORT]; int32_t sid; int32_t vgid; uint64_t uid; SSchema schema[]; -} STableMeta; +} STableMetaMsg; typedef struct SMultiTableMeta { int32_t numOfTables; int32_t contLen; - STableMeta metas[]; + STableMetaMsg metas[]; } SMultiTableMeta; typedef struct { @@ -718,9 +720,9 @@ typedef struct { char payload[]; } SCMShowMsg; -typedef struct { - uint64_t qhandle; - STableMeta tableMeta; +typedef struct SCMShowRsp { + uint64_t qhandle; + STableMetaMsg tableMeta; } SCMShowRsp; typedef struct { diff --git a/src/mnode/inc/mgmtChildTable.h b/src/mnode/inc/mgmtChildTable.h index 9ba5da50241f67c756dc6418de368be996fc079c..b16dd58f67402e84c2c611282fded15abea88e97 100644 --- a/src/mnode/inc/mgmtChildTable.h +++ b/src/mnode/inc/mgmtChildTable.h @@ -36,7 +36,7 @@ void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pCreate, SChildTableObj *p int32_t mgmtDropChildTable(SQueuedMsg *newMsg, SChildTableObj *pTable); int32_t mgmtModifyChildTableTagValueByName(SChildTableObj *pTable, char *tagName, char *nContent); -int32_t mgmtGetChildTableMeta(SDbObj *pDb, SChildTableObj *pTable, STableMeta *pMeta, bool usePublicIp); +int32_t mgmtGetChildTableMeta(SDbObj *pDb, SChildTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp); void mgmtDropAllChildTables(SDbObj *pDropDb); diff --git a/src/mnode/inc/mgmtNormalTable.h b/src/mnode/inc/mgmtNormalTable.h index c78d9fce43b27a5c470a188f44f27e65f1ce9b81..dd09a62bb4bc5f2051d374725e3759ddf5c2dae5 100644 --- a/src/mnode/inc/mgmtNormalTable.h +++ b/src/mnode/inc/mgmtNormalTable.h @@ -35,7 +35,7 @@ int32_t mgmtDropNormalTable(SQueuedMsg *newMsg, SNormalTableObj *pTable); int32_t mgmtAddNormalTableColumn(SNormalTableObj *pTable, SSchema schema[], int32_t ncols); int32_t mgmtDropNormalTableColumnByName(SNormalTableObj *pTable, char *colName); -int32_t mgmtGetNormalTableMeta(SDbObj *pDb, SNormalTableObj *pTable, STableMeta *pMeta, bool usePublicIp); +int32_t mgmtGetNormalTableMeta(SDbObj *pDb, SNormalTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp); void mgmtDropAllNormalTables(SDbObj *pDropDb); diff --git a/src/mnode/inc/mgmtShell.h b/src/mnode/inc/mgmtShell.h index a9af568547802fca47a913a46cddf94951a9af72..56625b982aed2fa90b39d35be7014ba7ed85e6a4 100644 --- a/src/mnode/inc/mgmtShell.h +++ b/src/mnode/inc/mgmtShell.h @@ -25,7 +25,7 @@ int32_t mgmtInitShell(); void mgmtCleanUpShell(); void mgmtAddShellMsgHandle(uint8_t msgType, void (*fp)(SQueuedMsg *queuedMsg)); -typedef int32_t (*SShowMetaFp)(STableMeta *pMeta, SShowObj *pShow, void *pConn); +typedef int32_t (*SShowMetaFp)(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); typedef int32_t (*SShowRetrieveFp)(SShowObj *pShow, char *data, int32_t rows, void *pConn); void mgmtAddShellShowMetaHandle(uint8_t showType, SShowMetaFp fp); void mgmtAddShellShowRetrieveHandle(uint8_t showType, SShowRetrieveFp fp); diff --git a/src/mnode/inc/mgmtSuperTable.h b/src/mnode/inc/mgmtSuperTable.h index 4b0599b359a660693ee8032940c8e9bc3042ab01..922aafed7f9344ad908893cb6c7e6278a541bf35 100644 --- a/src/mnode/inc/mgmtSuperTable.h +++ b/src/mnode/inc/mgmtSuperTable.h @@ -39,7 +39,7 @@ int32_t mgmtModifySuperTableTagNameByName(SSuperTableObj *pTable, char *oldTagNa int32_t mgmtAddSuperTableColumn(SSuperTableObj *pTable, SSchema schema[], int32_t ncols); int32_t mgmtDropSuperTableColumnByName(SSuperTableObj *pTable, char *colName); -int32_t mgmtGetSuperTableMeta(SDbObj *pDb, SSuperTableObj *pTable, STableMeta *pMeta, bool usePublicIp); +int32_t mgmtGetSuperTableMeta(SDbObj *pDb, SSuperTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp); void * mgmtGetSuperTableVgroup(SSuperTableObj *pStable); int32_t mgmtFindSuperTableTagIndex(SSuperTableObj *pTable, const char *tagName); diff --git a/src/mnode/inc/mgmtTable.h b/src/mnode/inc/mgmtTable.h index a6c537a49b51734dd42717cef75fa657892aea5a..b145210e9bbf1800ad6e80b46f635407102fb86c 100644 --- a/src/mnode/inc/mgmtTable.h +++ b/src/mnode/inc/mgmtTable.h @@ -30,7 +30,7 @@ void mgmtCleanUpTables(); STableInfo* mgmtGetTable(char *tableId); STableInfo* mgmtGetTableByPos(uint32_t dnodeIp, int32_t vnode, int32_t sid); -int32_t mgmtGetTableMeta(SDbObj *pDb, STableInfo *pTable, STableMeta *pMeta, bool usePublicIp); +int32_t mgmtGetTableMeta(SDbObj *pDb, STableInfo *pTable, STableMetaMsg *pMeta, bool usePublicIp); void mgmtAddTableIntoSuperTable(SSuperTableObj *pStable); void mgmtRemoveTableFromSuperTable(SSuperTableObj *pStable); diff --git a/src/mnode/src/mgmtChildTable.c b/src/mnode/src/mgmtChildTable.c index 5b27acf1c41c8746551e5837a9fa50221bc5cc7a..8383deee545253be00d9efe0010e198d1c92316a 100644 --- a/src/mnode/src/mgmtChildTable.c +++ b/src/mnode/src/mgmtChildTable.c @@ -16,8 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taosmsg.h" -#include "taosdef.h" -#include "tschemautil.h" #include "tscompression.h" #include "tskiplist.h" #include "ttime.h" @@ -412,7 +410,7 @@ int32_t mgmtModifyChildTableTagValueByName(SChildTableObj *pTable, char *tagName return 0; } -int32_t mgmtGetChildTableMeta(SDbObj *pDb, SChildTableObj *pTable, STableMeta *pMeta, bool usePublicIp) { +int32_t mgmtGetChildTableMeta(SDbObj *pDb, SChildTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp) { pMeta->uid = htobe64(pTable->uid); pMeta->sid = htonl(pTable->sid); pMeta->vgid = htonl(pTable->vgId); @@ -421,8 +419,8 @@ int32_t mgmtGetChildTableMeta(SDbObj *pDb, SChildTableObj *pTable, STableMeta *p pMeta->numOfTags = pTable->superTable->numOfTags; pMeta->numOfColumns = htons(pTable->superTable->numOfColumns); pMeta->tableType = pTable->type; - pMeta->contLen = sizeof(STableMeta) + mgmtSetSchemaFromSuperTable(pMeta->schema, pTable->superTable); - strcpy(pMeta->tableId, pTable->tableId); + pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromSuperTable(pMeta->schema, pTable->superTable); + strncpy(pMeta->tableId, pTable->tableId, tListLen(pTable->tableId)); SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { @@ -435,7 +433,6 @@ int32_t mgmtGetChildTableMeta(SDbObj *pDb, SChildTableObj *pTable, STableMeta *p pMeta->vpeerDesc[i].ip = pVgroup->vnodeGid[i].privateIp; } pMeta->vpeerDesc[i].vnode = htonl(pVgroup->vnodeGid[i].vnode); - pMeta->vpeerDesc[i].vgId = htonl(pVgroup->vgId); } pMeta->numOfVpeers = pVgroup->numOfVnodes; diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index 2549dd4c64511bc3fe8348ce09c926a1dfe46658..e502d82a19a489c2182ff4ca0ebe5006581de02d 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -15,14 +15,11 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "taoserror.h" -#include "tschemautil.h" -#include "tstatus.h" -#include "tutil.h" -#include "mnode.h" + +#include "mgmtDb.h" #include "mgmtAcct.h" #include "mgmtBalance.h" -#include "mgmtDb.h" +#include "mgmtChildTable.h" #include "mgmtDnode.h" #include "mgmtGrant.h" #include "mgmtShell.h" @@ -34,6 +31,12 @@ #include "mgmtTable.h" #include "mgmtUser.h" #include "mgmtVgroup.h" +#include "mnode.h" + +#include "taoserror.h" +#include "tstatus.h" +#include "tutil.h" +#include "name.h" static void *tsDbSdb = NULL; static int32_t tsDbUpdateSize; @@ -42,7 +45,7 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate); static void mgmtDropDb(void *handle, void *tmrId); static void mgmtSetDbDirty(SDbObj *pDb); -static int32_t mgmtGetDbMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn); +static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg); static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg); @@ -454,10 +457,10 @@ void mgmtCleanUpDbs() { sdbCloseTable(tsDbSdb); } -static int32_t mgmtGetDbMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn) { +static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { int32_t cols = 0; - SSchema *pSchema = tsGetSchema(pMeta); + SSchema *pSchema = pMeta->schema; SUserObj *pUser = mgmtGetUserFromConn(pConn); if (pUser == NULL) return 0; @@ -469,7 +472,7 @@ static int32_t mgmtGetDbMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn) { pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "created time"); + strcpy(pSchema[cols].name, "created_time"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index 77c314c56fad0a709f7ab34e61f263d5aea06235..bb5db070353ef300ea41c5cbc676404d522e8a58 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" #include "tmodule.h" -#include "tschemautil.h" #include "tstatus.h" #include "mgmtBalance.h" #include "mgmtDnode.h" @@ -39,11 +38,11 @@ void (*mgmtSetDnodeUnRemoveFp)(SDnodeObj *pDnode) = NULL; static SDnodeObj tsDnodeObj = {0}; static void * mgmtGetNextDnode(SShowObj *pShow, SDnodeObj **pDnode); static bool mgmtCheckConfigShow(SGlobalConfig *cfg); -static int32_t mgmtGetModuleMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn); +static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn); -static int32_t mgmtGetConfigMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn); +static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, void *pConn); -static int32_t mgmtGetVnodeMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn); +static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg); static void mgmtProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) ; @@ -74,7 +73,7 @@ bool mgmtCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType) { return status > 0; } -int32_t mgmtGetModuleMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn) { +int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { int32_t cols = 0; SUserObj *pUser = mgmtGetUserFromConn(pConn); @@ -82,7 +81,7 @@ int32_t mgmtGetModuleMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn) { if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_NO_RIGHTS; - SSchema *pSchema = tsGetSchema(pMeta); + SSchema *pSchema = pMeta->schema; pShow->bytes[cols] = 16; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; @@ -167,7 +166,7 @@ int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pCo return numOfRows; } -static int32_t mgmtGetConfigMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn) { +static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { int32_t cols = 0; SUserObj *pUser = mgmtGetUserFromConn(pConn); @@ -175,7 +174,7 @@ static int32_t mgmtGetConfigMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_NO_RIGHTS; - SSchema *pSchema = tsGetSchema(pMeta); + SSchema *pSchema = pMeta->schema; pShow->bytes[cols] = TSDB_CFG_OPTION_LEN; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; @@ -255,13 +254,13 @@ static int32_t mgmtRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, vo return numOfRows; } -static int32_t mgmtGetVnodeMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn) { +static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { int32_t cols = 0; SUserObj *pUser = mgmtGetUserFromConn(pConn); if (pUser == NULL) return 0; if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_NO_RIGHTS; - SSchema *pSchema = tsGetSchema(pMeta); + SSchema *pSchema = pMeta->schema; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; @@ -277,7 +276,7 @@ static int32_t mgmtGetVnodeMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn) pShow->bytes[cols] = 12; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "sync status"); + strcpy(pSchema[cols].name, "sync_status"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; diff --git a/src/mnode/src/mgmtMnode.c b/src/mnode/src/mgmtMnode.c index 8e459bf5afa8de8673a436d0299eb043c11e1636..3dfce5e6afa452558bdd1c522361684b13a147f6 100644 --- a/src/mnode/src/mgmtMnode.c +++ b/src/mnode/src/mgmtMnode.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" #include "trpc.h" -#include "tschemautil.h" #include "mgmtMnode.h" #include "mgmtSdb.h" #include "mgmtUser.h" @@ -31,7 +30,7 @@ void (*mpeerCleanUpMnodesFp)() = NULL; static SMnodeObj tsMnodeObj = {0}; static bool tsMnodeIsMaster = false; static bool tsMnodeIsServing = false; -static int32_t mgmtGetMnodeMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn); +static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); static char *mgmtMnodeStatusStr[] = { @@ -116,7 +115,7 @@ static void *mgmtGetNextMnode(SShowObj *pShow, SMnodeObj **pMnode) { return *pMnode; } -static int32_t mgmtGetMnodeMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn) { +static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { int32_t cols = 0; SUserObj *pUser = mgmtGetUserFromConn(pConn); @@ -124,7 +123,7 @@ static int32_t mgmtGetMnodeMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn) if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_NO_RIGHTS; - SSchema *pSchema = tsGetSchema(pMeta); + SSchema *pSchema = pMeta->schema; pShow->bytes[cols] = 16; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; diff --git a/src/mnode/src/mgmtNormalTable.c b/src/mnode/src/mgmtNormalTable.c index 58578faf9a3574a0981ffd6ba6266554ce617cf3..ce6c74aece2a35c8145cfb9743b11d67fd19f139 100644 --- a/src/mnode/src/mgmtNormalTable.c +++ b/src/mnode/src/mgmtNormalTable.c @@ -490,7 +490,7 @@ static int32_t mgmtSetSchemaFromNormalTable(SSchema *pSchema, SNormalTableObj *p return numOfCols * sizeof(SSchema); } -int32_t mgmtGetNormalTableMeta(SDbObj *pDb, SNormalTableObj *pTable, STableMeta *pMeta, bool usePublicIp) { +int32_t mgmtGetNormalTableMeta(SDbObj *pDb, SNormalTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp) { pMeta->uid = htobe64(pTable->uid); pMeta->sid = htonl(pTable->sid); pMeta->vgid = htonl(pTable->vgId); @@ -499,7 +499,9 @@ int32_t mgmtGetNormalTableMeta(SDbObj *pDb, SNormalTableObj *pTable, STableMeta pMeta->numOfTags = 0; pMeta->numOfColumns = htons(pTable->numOfColumns); pMeta->tableType = pTable->type; - pMeta->contLen = sizeof(STableMeta) + mgmtSetSchemaFromNormalTable(pMeta->schema, pTable); + pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromNormalTable(pMeta->schema, pTable); + + strncpy(pMeta->tableId, pTable->tableId, tListLen(pTable->tableId)); SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { @@ -511,8 +513,8 @@ int32_t mgmtGetNormalTableMeta(SDbObj *pDb, SNormalTableObj *pTable, STableMeta } else { pMeta->vpeerDesc[i].ip = pVgroup->vnodeGid[i].privateIp; } + pMeta->vpeerDesc[i].vnode = htonl(pVgroup->vnodeGid[i].vnode); - pMeta->vpeerDesc[i].vgId = htonl(pVgroup->vgId); } pMeta->numOfVpeers = pVgroup->numOfVnodes; diff --git a/src/mnode/src/mgmtProfile.c b/src/mnode/src/mgmtProfile.c index bb192ce8a56fd353da3dd9ce92046b3eab2154c2..b20e66dd874b6b589087b37e66f463e5d32aee89 100644 --- a/src/mnode/src/mgmtProfile.c +++ b/src/mnode/src/mgmtProfile.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taosmsg.h" -#include "tschemautil.h" #include "mgmtMnode.h" #include "mgmtProfile.h" #include "mgmtShell.h" @@ -137,10 +136,10 @@ int32_t mgmtGetQueries(SShowObj *pShow, void *pConn) { return 0; } -int32_t mgmtGetQueryMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn) { +int32_t mgmtGetQueryMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { int32_t cols = 0; - SSchema *pSchema = tsGetSchema(pMeta); + SSchema *pSchema = pMeta->schema; pShow->bytes[cols] = TSDB_USER_LEN; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; @@ -336,9 +335,9 @@ int32_t mgmtGetStreams(SShowObj *pShow, void *pConn) { return 0; } -int32_t mgmtGetStreamMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn) { +int32_t mgmtGetStreamMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { int32_t cols = 0; - SSchema *pSchema = tsGetSchema(pMeta); + SSchema *pSchema = pMeta->schema; pShow->bytes[cols] = TSDB_USER_LEN; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; @@ -600,11 +599,11 @@ int mgmtGetConns(SShowObj *pShow, void *pConn) { return 0; } -int32_t mgmtGetConnsMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn) { +int32_t mgmtGetConnsMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { int32_t cols = 0; pShow->bytes[cols] = TSDB_TABLE_NAME_LEN; - SSchema *pSchema = tsGetSchema(pMeta); + SSchema *pSchema = pMeta->schema; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "user"); diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index 501faaab97b1808e308799b71c71de56565a796a..3ff78b985e4cb55d07d3284f750a56bc4fb99faf 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -39,7 +39,7 @@ #include "mgmtUser.h" #include "mgmtVgroup.h" -typedef int32_t (*SShowMetaFp)(STableMeta *pMeta, SShowObj *pShow, void *pConn); +typedef int32_t (*SShowMetaFp)(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); typedef int32_t (*SShowRetrieveFp)(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int mgmtShellRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey); diff --git a/src/mnode/src/mgmtSuperTable.c b/src/mnode/src/mgmtSuperTable.c index de64f9f54e5e55866f684db5e9a804741029f681..b49b534aa9612ffbaed97654d53a90975058fd8e 100644 --- a/src/mnode/src/mgmtSuperTable.c +++ b/src/mnode/src/mgmtSuperTable.c @@ -15,15 +15,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "taosmsg.h" -#include "tschemautil.h" -#include "tscompression.h" -#include "tskiplist.h" -#include "tsqlfunction.h" -#include "ttime.h" -#include "tstatus.h" -#include "tutil.h" -#include "mnode.h" + #include "mgmtAcct.h" #include "mgmtChildTable.h" #include "mgmtDb.h" @@ -35,12 +27,16 @@ #include "mgmtTable.h" #include "mgmtUser.h" #include "mgmtVgroup.h" +#include "mnode.h" + +#include "name.h" +#include "tsqlfunction.h" static void *tsSuperTableSdb; static int32_t tsSuperTableUpdateSize; static int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, void *pConn); -static int32_t mgmtGetShowSuperTableMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn); +static int32_t mgmtGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static void mgmtDestroySuperTable(SSuperTableObj *pTable) { tfree(pTable->schema); @@ -458,14 +454,14 @@ int32_t mgmtDropSuperTableColumnByName(SSuperTableObj *pStable, char *colName) { return TSDB_CODE_SUCCESS; } -static int32_t mgmtGetShowSuperTableMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn) { +static int32_t mgmtGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { SDbObj *pDb = mgmtGetDb(pShow->db); if (pDb == NULL) { return TSDB_CODE_DB_NOT_SELECTED; } int32_t cols = 0; - SSchema *pSchema = tsGetSchema(pMeta); + SSchema *pSchema = pMeta->schema; pShow->bytes[cols] = TSDB_TABLE_NAME_LEN; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; @@ -475,7 +471,7 @@ static int32_t mgmtGetShowSuperTableMeta(STableMeta *pMeta, SShowObj *pShow, voi pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create time"); + strcpy(pSchema[cols].name, "create_time"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -623,7 +619,7 @@ int32_t mgmtSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTable) { return (pTable->numOfColumns + pTable->numOfTags) * sizeof(SSchema); } -int32_t mgmtGetSuperTableMeta(SDbObj *pDb, SSuperTableObj *pTable, STableMeta *pMeta, bool usePublicIp) { +int32_t mgmtGetSuperTableMeta(SDbObj *pDb, SSuperTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp) { pMeta->uid = htobe64(pTable->uid); pMeta->sid = htonl(pTable->sid); pMeta->vgid = htonl(pTable->vgId); @@ -632,7 +628,7 @@ int32_t mgmtGetSuperTableMeta(SDbObj *pDb, SSuperTableObj *pTable, STableMeta *p pMeta->numOfTags = pTable->numOfTags; pMeta->numOfColumns = htons(pTable->numOfColumns); pMeta->tableType = pTable->type; - pMeta->contLen = sizeof(STableMeta) + mgmtSetSchemaFromSuperTable(pMeta->schema, pTable); + pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromSuperTable(pMeta->schema, pTable); strcpy(pMeta->tableId, pTable->tableId); return TSDB_CODE_SUCCESS; diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index a907cd85f53da70b69c2c0ff6e9130f3212770e8..255b2a6f27035ccf4858494a915057090e9190a6 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -14,6 +14,8 @@ */ #define _DEFAULT_SOURCE +#include "os.h" + #include "mgmtTable.h" #include "mgmtAcct.h" #include "mgmtChildTable.h" @@ -30,17 +32,17 @@ #include "mgmtUser.h" #include "mgmtVgroup.h" #include "mnode.h" -#include "os.h" + #include "qast.h" #include "qextbuffer.h" #include "taoserror.h" #include "taosmsg.h" -#include "tschemautil.h" #include "tscompression.h" #include "tskiplist.h" #include "tsqlfunction.h" #include "tstatus.h" #include "ttime.h" +#include "name.h" extern void *tsNormalTableSdb; extern void *tsChildTableSdb; @@ -53,9 +55,10 @@ static void mgmtProcessMultiTableMetaMsg(SQueuedMsg *queueMsg); static void mgmtProcessSuperTableMetaMsg(SQueuedMsg *queueMsg); static void mgmtProcessCreateTableRsp(SRpcMsg *rpcMsg); static void mgmtProcessDropTableRsp(SRpcMsg *rpcMsg); +static int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static void mgmtProcessAlterTableRsp(SRpcMsg *rpcMsg); static void mgmtProcessDropStableRsp(SRpcMsg *rpcMsg); -static int32_t mgmtGetShowTableMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn); +static int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mgmtProcessGetTableMeta(STableInfo *pTable, void *thandle); @@ -123,7 +126,7 @@ STableInfo* mgmtGetTableByPos(uint32_t dnodeId, int32_t vnode, int32_t sid) { return pVgroup->tableList[sid]; } -int32_t mgmtGetTableMeta(SDbObj *pDb, STableInfo *pTable, STableMeta *pMeta, bool usePublicIp) { +int32_t mgmtGetTableMeta(SDbObj *pDb, STableInfo *pTable, STableMetaMsg *pMeta, bool usePublicIp) { if (pTable->type == TSDB_CHILD_TABLE) { mgmtGetChildTableMeta(pDb, (SChildTableObj *) pTable, pMeta, usePublicIp); } else if (pTable->type == TSDB_NORMAL_TABLE) { @@ -188,24 +191,24 @@ void mgmtCleanUpTables() { mgmtCleanUpSuperTables(); } -int32_t mgmtGetShowTableMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn) { +int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { SDbObj *pDb = mgmtGetDb(pShow->db); if (pDb == NULL) { return TSDB_CODE_DB_NOT_SELECTED; } int32_t cols = 0; - SSchema *pSchema = tsGetSchema(pMeta); + SSchema *pSchema = pMeta->schema; pShow->bytes[cols] = TSDB_TABLE_NAME_LEN; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "table name"); + strcpy(pSchema[cols].name, "table_name"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create time"); + strcpy(pSchema[cols].name, "created_time"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -217,7 +220,7 @@ int32_t mgmtGetShowTableMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn) { pShow->bytes[cols] = TSDB_TABLE_NAME_LEN; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "super table name"); + strcpy(pSchema[cols].name, "stable_name"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -602,16 +605,17 @@ void mgmtProcessGetTableMeta(STableInfo *pTable, void *thandle) { } bool usePublicIp = (connInfo.serverIp == tsPublicIpInt); - - STableMeta *pMeta = rpcMallocCont(sizeof(STableMeta) + sizeof(SSchema) * TSDB_MAX_COLUMNS); + + STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS); rpcRsp.code = mgmtGetTableMeta(pDb, pTable, pMeta, usePublicIp); if (rpcRsp.code != TSDB_CODE_SUCCESS) { rpcFreeCont(pMeta); } else { - pMeta->contLen = htons(pMeta->contLen); rpcRsp.pCont = pMeta; rpcRsp.contLen = pMeta->contLen; + + pMeta->contLen = htons(pMeta->contLen); } rpcSendResponse(&rpcRsp); @@ -696,7 +700,7 @@ void mgmtProcessMultiTableMetaMsg(SQueuedMsg *pMsg) { if (pDb == NULL) continue; int availLen = totalMallocLen - pMultiMeta->contLen; - if (availLen <= sizeof(STableMeta) + sizeof(SSchema) * TSDB_MAX_COLUMNS) { + if (availLen <= sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS) { //TODO realloc //totalMallocLen *= 2; //pMultiMeta = rpcReMalloc(pMultiMeta, totalMallocLen); @@ -709,7 +713,7 @@ void mgmtProcessMultiTableMetaMsg(SQueuedMsg *pMsg) { //} } - STableMeta *pMeta = (STableMeta *)(pMultiMeta->metas + pMultiMeta->contLen); + STableMetaMsg *pMeta = (STableMetaMsg *)(pMultiMeta->metas + pMultiMeta->contLen); int32_t code = mgmtGetTableMeta(pDb, pTable, pMeta, usePublicIp); if (code == TSDB_CODE_SUCCESS) { pMultiMeta->numOfTables ++; diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index 3321c415136ab602cc6519b96421b039a4a5b47b..6e72a6c8c2de4a95f23c6ac5fccb6d22e1f46438 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" #include "trpc.h" -#include "tschemautil.h" #include "ttime.h" #include "tutil.h" #include "mgmtAcct.h" @@ -32,7 +31,7 @@ static int32_t tsUserUpdateSize = 0; static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass); static int32_t mgmtDropUser(SAcctObj *pAcct, char *name); static int32_t mgmtUpdateUser(SUserObj *pUser); -static int32_t mgmtGetUserMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn); +static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mgmtProcessCreateUserMsg(SQueuedMsg *pMsg); @@ -214,14 +213,14 @@ static int32_t mgmtDropUser(SAcctObj *pAcct, char *name) { return 0; } -static int32_t mgmtGetUserMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn) { +static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { SUserObj *pUser = mgmtGetUserFromConn(pConn); if (pUser == NULL) { return TSDB_CODE_INVALID_USER; } int32_t cols = 0; - SSchema *pSchema = tsGetSchema(pMeta); + SSchema *pSchema = pMeta->schema; pShow->bytes[cols] = TSDB_USER_LEN; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; @@ -237,7 +236,7 @@ static int32_t mgmtGetUserMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn) pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "created time"); + strcpy(pSchema[cols].name, "created_time"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index 12ac6fef57b8a99b904f5ca00594620a724e5cdd..5b1968259f3544712f90bfff1ed41542f2e94001 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -17,7 +17,6 @@ #include "os.h" #include "taoserror.h" #include "tlog.h" -#include "tschemautil.h" #include "tstatus.h" #include "mnode.h" #include "mgmtBalance.h" @@ -33,7 +32,7 @@ static void *tsVgroupSdb = NULL; static int32_t tsVgUpdateSize = 0; -static int32_t mgmtGetVgroupMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn); +static int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg); static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg); @@ -265,14 +264,14 @@ void mgmtCleanUpVgroups() { sdbCloseTable(tsVgroupSdb); } -int32_t mgmtGetVgroupMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn) { +int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { SDbObj *pDb = mgmtGetDb(pShow->db); if (pDb == NULL) { return TSDB_CODE_DB_NOT_SELECTED; } int32_t cols = 0; - SSchema *pSchema = tsGetSchema(pMeta); + SSchema *pSchema = pMeta->schema; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; @@ -288,7 +287,7 @@ int32_t mgmtGetVgroupMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn) { pShow->bytes[cols] = 9; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "vgroup status"); + strcpy(pSchema[cols].name, "vgroup_status"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -328,13 +327,13 @@ int32_t mgmtGetVgroupMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn) { pShow->bytes[cols] = 9; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "vnode status"); + strcpy(pSchema[cols].name, "vnode_status"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 16; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "public ip"); + strcpy(pSchema[cols].name, "public_ip"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; } @@ -371,7 +370,7 @@ char *mgmtGetVnodeStatus(SVgObj *pVgroup, SVnodeGid *pVnode) { for (int i = 0; i < pDnode->openVnodes; ++i) { if (pDnode->vload[i].vgId == pVgroup->vgId) { - return (char*)taosGetVnodeStatusStr(pDnode->vload[i].status); + return (char*)taosGetVnodeStatusStr(pDnode->vload[i].status); } } @@ -488,7 +487,6 @@ SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) { SVnodeDesc *vpeerDesc = pVnode->vpeerDesc; for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) { - vpeerDesc[j].vgId = htonl(pVgroup->vgId); vpeerDesc[j].ip = htonl(pVgroup->vnodeGid[j].privateIp); } @@ -659,8 +657,8 @@ void mgmtUpdateVgroupIp(SDnodeObj *pDnode) { for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { SVnodeGid *vnodeGid = pVgroup->vnodeGid + i; if (vnodeGid->dnodeId == pDnode->dnodeId) { - mPrint("vgroup:%d, dnode:%d, privateIp:%s change to %s, publicIp:%s change to %s", - pVgroup->vgId, vnodeGid->dnodeId, pDnode->privateIp, taosIpStr(vnodeGid->privateIp), + mPrint("vgroup:%d, dnode:%d, privateIp:%s change to %s, publicIp:%s change to %s", + pVgroup->vgId, vnodeGid->dnodeId, pDnode->privateIp, taosIpStr(vnodeGid->privateIp), pDnode->publicIp, taosIpStr(vnodeGid->publicIp)); vnodeGid->publicIp = pDnode->publicIp; vnodeGid->privateIp = pDnode->privateIp; diff --git a/src/query/CMakeLists.txt b/src/query/CMakeLists.txt index c0ded1dafb70cf36ea05886cbe17904b18addc24..0984aeb585df2f00d23853c6c88498e8ca14b29a 100644 --- a/src/query/CMakeLists.txt +++ b/src/query/CMakeLists.txt @@ -4,10 +4,12 @@ PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/vnode/tsdb/inc) INCLUDE_DIRECTORIES(inc) IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(query ${SRC}) - TARGET_LINK_LIBRARIES(query util m rt) + TARGET_LINK_LIBRARIES(query tutil m rt) ENDIF () \ No newline at end of file diff --git a/src/inc/qpercentile.h b/src/query/inc/qpercentile.h similarity index 95% rename from src/inc/qpercentile.h rename to src/query/inc/qpercentile.h index 73430bd05ccb07df93c1fc9869539b504c6e618e..f8f09357c635afd096cab622daab186eb2fbc512 100644 --- a/src/inc/qpercentile.h +++ b/src/query/inc/qpercentile.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TPERCENTILE_H -#define TDENGINE_TPERCENTILE_H +#ifndef TDENGINE_QPERCENTILE_H +#define TDENGINE_QPERCENTILE_H #include "qextbuffer.h" @@ -74,4 +74,4 @@ void tBucketIntHash(tMemBucket *pBucket, void *value, int16_t *segIdx, int16_t * void tBucketDoubleHash(tMemBucket *pBucket, void *value, int16_t *segIdx, int16_t *slotIdx); -#endif // TDENGINE_TPERCENTILE_H +#endif // TDENGINE_QPERCENTILE_H diff --git a/src/query/inc/qresultBuf.h b/src/query/inc/qresultBuf.h index 346dc2d00c66e33cf63cffdb3c748cfecad02832..39600f512963745a90bd082bdfc3c412d38374c7 100644 --- a/src/query/inc/qresultBuf.h +++ b/src/query/inc/qresultBuf.h @@ -29,7 +29,7 @@ typedef struct SIDList { int32_t* pData; } SIDList; -typedef struct SQueryDiskbasedResultBuf { +typedef struct SDiskbasedResultBuf { int32_t numOfRowsPerPage; int32_t numOfPages; int64_t totalBufSize; @@ -42,7 +42,7 @@ typedef struct SQueryDiskbasedResultBuf { uint32_t numOfAllocGroupIds; // number of allocated id list void* idsTable; // id hash table SIDList* list; // for each id, there is a page id list -} SQueryDiskbasedResultBuf; +} SDiskbasedResultBuf; /** * create disk-based result buffer @@ -51,7 +51,7 @@ typedef struct SQueryDiskbasedResultBuf { * @param rowSize * @return */ -int32_t createDiskbasedResultBuffer(SQueryDiskbasedResultBuf** pResultBuf, int32_t size, int32_t rowSize); +int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t size, int32_t rowSize); /** * @@ -60,14 +60,14 @@ int32_t createDiskbasedResultBuffer(SQueryDiskbasedResultBuf** pResultBuf, int32 * @param pageId * @return */ -tFilePage* getNewDataBuf(SQueryDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId); +tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId); /** * * @param pResultBuf * @return */ -int32_t getNumOfRowsPerPage(SQueryDiskbasedResultBuf* pResultBuf); +int32_t getNumOfRowsPerPage(SDiskbasedResultBuf* pResultBuf); /** * @@ -75,7 +75,7 @@ int32_t getNumOfRowsPerPage(SQueryDiskbasedResultBuf* pResultBuf); * @param groupId * @return */ -SIDList getDataBufPagesIdList(SQueryDiskbasedResultBuf* pResultBuf, int32_t groupId); +SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId); /** * get the specified buffer page by id @@ -83,27 +83,27 @@ SIDList getDataBufPagesIdList(SQueryDiskbasedResultBuf* pResultBuf, int32_t grou * @param id * @return */ -tFilePage* getResultBufferPageById(SQueryDiskbasedResultBuf* pResultBuf, int32_t id); +tFilePage* getResultBufferPageById(SDiskbasedResultBuf* pResultBuf, int32_t id); /** * get the total buffer size in the format of disk file * @param pResultBuf * @return */ -int32_t getResBufSize(SQueryDiskbasedResultBuf* pResultBuf); +int32_t getResBufSize(SDiskbasedResultBuf* pResultBuf); /** * get the number of groups in the result buffer * @param pResultBuf * @return */ -int32_t getNumOfResultBufGroupId(SQueryDiskbasedResultBuf* pResultBuf); +int32_t getNumOfResultBufGroupId(SDiskbasedResultBuf* pResultBuf); /** * destroy result buffer * @param pResultBuf */ -void destroyResultBuf(SQueryDiskbasedResultBuf* pResultBuf); +void destroyResultBuf(SDiskbasedResultBuf* pResultBuf); /** * diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h new file mode 100644 index 0000000000000000000000000000000000000000..c3465d361a175356fb0495fb1672bd71ba8da1f7 --- /dev/null +++ b/src/query/inc/queryExecutor.h @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef TDENGINE_QUERYEXECUTOR_H +#define TDENGINE_QUERYEXECUTOR_H + +#include "os.h" + +#include "hash.h" +#include "qinterpolation.h" +#include "qresultBuf.h" +#include "qsqlparser.h" +#include "qtsbuf.h" +#include "taosdef.h" +#include "tref.h" +#include "tsqlfunction.h" + +typedef struct SData { + int32_t num; + char data[]; +} SData; + +enum { + ST_QUERY_KILLED = 0, // query killed + ST_QUERY_PAUSED = 1, // query paused, due to full of the response buffer + ST_QUERY_COMPLETED = 2, // query completed +}; + +struct SColumnFilterElem; +typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2); +typedef int (*__block_search_fn_t)(char* data, int num, int64_t key, int order); + +typedef struct SSqlGroupbyExpr { + int16_t tableIndex; + int16_t numOfGroupCols; + SColIndexEx columnInfo[TSDB_MAX_TAGS]; // group by columns information + int16_t orderIndex; // order by column index + int16_t orderType; // order by type: asc/desc +} SSqlGroupbyExpr; + +typedef struct SPosInfo { + int16_t pageId; + int16_t rowId; +} SPosInfo; + +typedef struct SWindowStatus { + bool closed; +} SWindowStatus; + +typedef struct SWindowResult { + uint16_t numOfRows; + SPosInfo pos; // Position of current result in disk-based output buffer + SResultInfo* resultInfo; // For each result column, there is a resultInfo + STimeWindow window; // The time window that current result covers. + SWindowStatus status; +} SWindowResult; + +typedef struct SResultRec { + int64_t pointsTotal; + int64_t pointsRead; +} SResultRec; + +typedef struct SWindowResInfo { + SWindowResult* pResult; // result list + void* hashList; // hash list for quick access + int16_t type; // data type for hash key + int32_t capacity; // max capacity + int32_t curIndex; // current start active index + int32_t size; // number of result set + int64_t startTime; // start time of the first time window for sliding query + int64_t prevSKey; // previous (not completed) sliding window start key + int64_t threshold; // threshold to pausing query and return closed results. +} SWindowResInfo; + +typedef struct SColumnFilterElem { + int16_t bytes; // column length + __filter_func_t fp; + SColumnFilterInfo filterInfo; +} SColumnFilterElem; + +typedef struct SSingleColumnFilterInfo { + SColumnInfoEx info; + int32_t numOfFilters; + SColumnFilterElem* pFilters; + void* pData; +} SSingleColumnFilterInfo; + +/* intermediate pos during multimeter query involves interval */ +typedef struct STableQueryInfo { + int64_t lastKey; + STimeWindow win; + int32_t numOfRes; + int16_t queryRangeSet; // denote if the query range is set, only available for interval query + int64_t tag; + STSCursor cur; + int32_t sid; // for retrieve the page id list + + SWindowResInfo windowResInfo; +} STableQueryInfo; + +typedef struct STableDataInfo { + int32_t numOfBlocks; + int32_t start; // start block index + int32_t tableIndex; + void* pMeterObj; + int32_t groupIdx; // group id in table list + STableQueryInfo* pTableQInfo; +} STableDataInfo; + +typedef struct SQuery { + int16_t numOfCols; + SOrderVal order; + STimeWindow window; + int64_t intervalTime; + int64_t slidingTime; // sliding time for sliding window query + char slidingTimeUnit; // interval data type, used for daytime revise + int8_t precision; + int16_t numOfOutputCols; + int16_t interpoType; + int16_t checkBufferInLoop; // check if the buffer is full during scan each block + SLimitVal limit; + int32_t rowSize; + SSqlGroupbyExpr* pGroupbyExpr; + SSqlFunctionExpr* pSelectExpr; + SColumnInfoEx* colList; + int32_t numOfFilterCols; + int64_t* defaultVal; + TSKEY lastKey; + uint32_t status; // query status + SResultRec rec; + int32_t pos; + int64_t pointsOffset; // the number of points offset to save read data + SData** sdata; + + SSingleColumnFilterInfo* pFilterInfo; +} SQuery; + +typedef struct SQueryCostSummary { +} SQueryCostSummary; + +typedef struct SQueryRuntimeEnv { + SResultInfo* resultInfo; // todo refactor to merge with SWindowResInfo + SQuery* pQuery; + void* pTabObj; + SData** pInterpoBuf; + SQLFunctionCtx* pCtx; + int16_t numOfRowsPerPage; + int16_t offset[TSDB_MAX_COLUMNS]; + uint16_t scanFlag; // denotes reversed scan of data or not + SInterpolationInfo interpoInfo; + SWindowResInfo windowResInfo; + STSBuf* pTSBuf; + STSCursor cur; + SQueryCostSummary summary; + bool stableQuery; // super table query or not + void* pQueryHandle; + + SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file +} SQueryRuntimeEnv; + +typedef struct SQInfo { + uint64_t signature; + TSKEY startTime; + int64_t elapsedTime; + SResultRec rec; + int pointsReturned; + int pointsInterpo; + int code; // error code to returned to client + sem_t dataReady; + SHashObj* pTableList; // table list + SQueryRuntimeEnv runtimeEnv; + int32_t subgroupIdx; + int32_t offset; /* offset in group result set of subgroup */ + tSidSet* pSidSet; + + T_REF_DECLARE() + /* + * the query is executed position on which meter of the whole list. + * when the index reaches the last one of the list, it means the query is completed. + * We later may refactor to remove this attribution by using another flag to denote + * whether a multimeter query is completed or not. + */ + int32_t tableIndex; + int32_t numOfGroupResultPages; + STableDataInfo* pTableDataInfo; + TSKEY* tsList; +} SQInfo; + +/** + * create the qinfo object before adding the query task to each tsdb query worker + * + * @param pReadMsg + * @param pQInfo + * @return + */ +int32_t qCreateQueryInfo(void* pReadMsg, SQInfo** pQInfo); + +/** + * query on single table + * @param pReadMsg + */ +void qTableQuery(void* pReadMsg); + +/** + * query on super table + * @param pReadMsg + */ +void qSuperTableQuery(void* pReadMsg); + +#endif // TDENGINE_QUERYEXECUTOR_H diff --git a/src/query/inc/queryUtil.h b/src/query/inc/queryUtil.h new file mode 100644 index 0000000000000000000000000000000000000000..06a716b750599881824d8096b9b01c0a90760771 --- /dev/null +++ b/src/query/inc/queryUtil.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef TDENGINE_QUERYUTIL_H +#define TDENGINE_QUERYUTIL_H + +void clearTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* pOneOutputRes); +void copyTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* dst, const SWindowResult* src); + +int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, SQueryRuntimeEnv* pRuntimeEnv, int32_t size, + int32_t threshold, int16_t type); + +void cleanupTimeWindowInfo(SWindowResInfo* pWindowResInfo, int32_t numOfCols); +void resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo* pWindowResInfo); +void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num); + +void clearClosedTimeWindow(SQueryRuntimeEnv* pRuntimeEnv); +int32_t numOfClosedTimeWindow(SWindowResInfo* pWindowResInfo); +void closeTimeWindow(SWindowResInfo* pWindowResInfo, int32_t slot); +void closeAllTimeWindow(SWindowResInfo* pWindowResInfo); +void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_t order); +SWindowResult *getWindowResult(SWindowResInfo *pWindowResInfo, int32_t slot); +int32_t curTimeWindow(SWindowResInfo *pWindowResInfo); +bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot); + +void createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, SPosInfo *posInfo); + +#endif // TDENGINE_QUERYUTIL_H diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 27bce71300ccf17743eaa50d64a975b624bde5d2..04ff9e56f8238ca393ed50a4e975191d158517dd 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -20,11 +20,11 @@ extern "C" { #endif -#include -#include +#include "os.h" -#include "trpc.h" +#include "../../common/inc/name.h" #include "taosdef.h" +#include "trpc.h" #include "tvariant.h" #define TSDB_FUNC_INVALID_ID -1 @@ -130,12 +130,8 @@ typedef struct SArithmeticSupport { typedef struct SQLPreAggVal { bool isSet; - int32_t numOfNull; - int64_t sum; - int64_t max; - int64_t min; - int16_t maxIndex; - int16_t minIndex; + int32_t size; + SDataStatis statis; } SQLPreAggVal; typedef struct SInterpInfoDetail { diff --git a/src/query/src/qast.c b/src/query/src/qast.c index ba8031578fd5ea7194b1e192026c1181a2746a61..a0cbf121691c5e341964091038d233b40f1c3c58 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -20,7 +20,6 @@ #include "taosdef.h" #include "taosmsg.h" #include "tlog.h" -#include "tschemautil.h" #include "tsqlfunction.h" #include "tstoken.h" #include "ttokendef.h" diff --git a/src/query/src/qresultBuf.c b/src/query/src/qresultBuf.c index fa7c59be4e692885f812075f034936cb37e5d3ce..07fd5f11e3c2da455f8639cc69ba0f74af2fc18d 100644 --- a/src/query/src/qresultBuf.c +++ b/src/query/src/qresultBuf.c @@ -7,8 +7,8 @@ #define DEFAULT_INTERN_BUF_SIZE 16384L -int32_t createDiskbasedResultBuffer(SQueryDiskbasedResultBuf** pResultBuf, int32_t size, int32_t rowSize) { - SQueryDiskbasedResultBuf* pResBuf = calloc(1, sizeof(SQueryDiskbasedResultBuf)); +int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t size, int32_t rowSize) { + SDiskbasedResultBuf* pResBuf = calloc(1, sizeof(SDiskbasedResultBuf)); pResBuf->numOfRowsPerPage = (DEFAULT_INTERN_BUF_SIZE - sizeof(tFilePage)) / rowSize; pResBuf->numOfPages = size; @@ -50,17 +50,17 @@ int32_t createDiskbasedResultBuffer(SQueryDiskbasedResultBuf** pResultBuf, int32 return TSDB_CODE_SUCCESS; } -tFilePage* getResultBufferPageById(SQueryDiskbasedResultBuf* pResultBuf, int32_t id) { +tFilePage* getResultBufferPageById(SDiskbasedResultBuf* pResultBuf, int32_t id) { assert(id < pResultBuf->numOfPages && id >= 0); return (tFilePage*)(pResultBuf->pBuf + DEFAULT_INTERN_BUF_SIZE * id); } -int32_t getNumOfResultBufGroupId(SQueryDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->idsTable); } +int32_t getNumOfResultBufGroupId(SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->idsTable); } -int32_t getResBufSize(SQueryDiskbasedResultBuf* pResultBuf) { return pResultBuf->totalBufSize; } +int32_t getResBufSize(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->totalBufSize; } -static int32_t extendDiskFileSize(SQueryDiskbasedResultBuf* pResultBuf, int32_t numOfPages) { +static int32_t extendDiskFileSize(SDiskbasedResultBuf* pResultBuf, int32_t numOfPages) { assert(pResultBuf->numOfPages * DEFAULT_INTERN_BUF_SIZE == pResultBuf->totalBufSize); int32_t ret = munmap(pResultBuf->pBuf, pResultBuf->totalBufSize); @@ -88,11 +88,11 @@ static int32_t extendDiskFileSize(SQueryDiskbasedResultBuf* pResultBuf, int32_t return TSDB_CODE_SUCCESS; } -static bool noMoreAvailablePages(SQueryDiskbasedResultBuf* pResultBuf) { +static bool noMoreAvailablePages(SDiskbasedResultBuf* pResultBuf) { return (pResultBuf->allocateId == pResultBuf->numOfPages - 1); } -static int32_t getGroupIndex(SQueryDiskbasedResultBuf* pResultBuf, int32_t groupId) { +static int32_t getGroupIndex(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { assert(pResultBuf != NULL); char* p = taosHashGet(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t)); @@ -106,7 +106,7 @@ static int32_t getGroupIndex(SQueryDiskbasedResultBuf* pResultBuf, int32_t group return slot; } -static int32_t addNewGroupId(SQueryDiskbasedResultBuf* pResultBuf, int32_t groupId) { +static int32_t addNewGroupId(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { int32_t num = getNumOfResultBufGroupId(pResultBuf); // the num is the newest allocated group id slot if (pResultBuf->numOfAllocGroupIds <= num) { @@ -148,7 +148,7 @@ static int32_t doRegisterId(SIDList* pList, int32_t id) { return 0; } -static void registerPageId(SQueryDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) { +static void registerPageId(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) { int32_t slot = getGroupIndex(pResultBuf, groupId); if (slot < 0) { slot = addNewGroupId(pResultBuf, groupId); @@ -158,7 +158,7 @@ static void registerPageId(SQueryDiskbasedResultBuf* pResultBuf, int32_t groupId doRegisterId(pList, pageId); } -tFilePage* getNewDataBuf(SQueryDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) { +tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) { if (noMoreAvailablePages(pResultBuf)) { if (extendDiskFileSize(pResultBuf, pResultBuf->incStep) != TSDB_CODE_SUCCESS) { return NULL; @@ -177,9 +177,9 @@ tFilePage* getNewDataBuf(SQueryDiskbasedResultBuf* pResultBuf, int32_t groupId, return page; } -int32_t getNumOfRowsPerPage(SQueryDiskbasedResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; } +int32_t getNumOfRowsPerPage(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; } -SIDList getDataBufPagesIdList(SQueryDiskbasedResultBuf* pResultBuf, int32_t groupId) { +SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { SIDList list = {0}; int32_t slot = getGroupIndex(pResultBuf, groupId); if (slot < 0) { @@ -189,7 +189,7 @@ SIDList getDataBufPagesIdList(SQueryDiskbasedResultBuf* pResultBuf, int32_t grou } } -void destroyResultBuf(SQueryDiskbasedResultBuf* pResultBuf) { +void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { if (pResultBuf == NULL) { return; } diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c new file mode 100644 index 0000000000000000000000000000000000000000..4f7e08b5633d844b08f5e57e6b8fa4d6737caebb --- /dev/null +++ b/src/query/src/queryExecutor.c @@ -0,0 +1,5397 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "os.h" + +#include "hash.h" +#include "hashfunc.h" +#include "taosmsg.h" +#include "tlosertree.h" +#include "tstatus.h" +#include "tscompression.h" +#include "ttime.h" +#include "tlog.h" + +#include "queryExecutor.h" +#include "queryUtil.h" +#include "tsdb.h" +#include "qresultBuf.h" + +#define DEFAULT_INTERN_BUF_SIZE 16384L + +/** + * check if the primary column is load by default, otherwise, the program will + * forced to load primary column explicitly. + */ +#define PRIMARY_TSCOL_LOADED(query) ((query)->colList[0].data.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) + + +#define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0) +#define TSDB_COL_IS_TAG(f) (((f)&TSDB_COL_TAG) != 0) +#define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP) + +#define IS_MASTER_SCAN(runtime) (((runtime)->scanFlag & 1u) == MASTER_SCAN) +#define IS_SUPPLEMENT_SCAN(runtime) ((runtime)->scanFlag == SUPPLEMENTARY_SCAN) +#define SET_SUPPLEMENT_SCAN_FLAG(runtime) ((runtime)->scanFlag = SUPPLEMENTARY_SCAN) +#define SET_MASTER_SCAN_FLAG(runtime) ((runtime)->scanFlag = MASTER_SCAN) + +#define GET_QINFO_ADDR(x) ((char*)(x)-offsetof(SQInfo, runtimeEnv)) + +#define GET_COL_DATA_POS(query, index, step) ((query)->pos + (index)*(step)) + +/* get the qinfo struct address from the query struct address */ +#define GET_COLUMN_BYTES(query, colidx) \ + ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIdxInBuf].data.bytes) +#define GET_COLUMN_TYPE(query, colidx) \ + ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIdxInBuf].data.type) + + +typedef struct SPointInterpoSupporter { + int32_t numOfCols; + char ** pPrevPoint; + char ** pNextPoint; +} SPointInterpoSupporter; + +typedef enum { + + /* + * the program will call this function again, if this status is set. + * used to transfer from QUERY_RESBUF_FULL + */ + QUERY_NOT_COMPLETED = 0x1u, + + /* + * output buffer is full, so, the next query will be employed, + * in this case, we need to set the appropriated start scan point for + * the next query. + * + * this status is only exist in group-by clause and + * diff/add/division/multiply/ query. + */ + QUERY_RESBUF_FULL = 0x2u, + + /* + * query is over + * 1. this status is used in one row result query process, e.g., + * count/sum/first/last/ + * avg...etc. + * 2. when the query range on timestamp is satisfied, it is also denoted as + * query_compeleted + */ + QUERY_COMPLETED = 0x4u, + + /* + * all data has been scanned, so current search is stopped, + * At last, the function will transfer this status to QUERY_COMPLETED + */ + QUERY_NO_DATA_TO_CHECK = 0x8u, +} vnodeQueryStatus; + + +static void setQueryStatus(SQuery *pQuery, int8_t status); +bool isIntervalQuery(SQuery *pQuery) { return pQuery->intervalTime > 0; } + +int32_t setQueryCtxForTableQuery(void* pReadMsg, SQInfo** pQInfo) { + +} + +enum { + TS_JOIN_TS_EQUAL = 0, + TS_JOIN_TS_NOT_EQUALS = 1, + TS_JOIN_TAG_NOT_EQUALS = 2, +}; + +static int32_t doMergeMetersResultsToGroupRes(SQInfo *pQInfo, STableDataInfo *pTableDataInfo, int32_t start, int32_t end); + +static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult); + +static void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pResultInfo); +static int32_t flushFromResultBuf(SQInfo *pQInfo); +static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId); +static void getNextTimeWindow(SQuery *pQuery, STimeWindow *pTimeWindow); + +static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, char *primaryColumnData, int32_t size, + int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t scanFlag); +static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); +static void destroyMeterQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols); +static int32_t setAdditionalInfo(SQInfo *pQInfo, int32_t meterIdx, STableQueryInfo *pTableQueryInfo); +static void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); +static bool hasMainOutput(SQuery *pQuery); + +bool getNeighborPoints(SQInfo *pQInfo, void *pMeterObj, SPointInterpoSupporter *pPointInterpSupporter) { +#if 0 + SQueryRuntimeEnv *pRuntimeEnv = &pSupporter->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + + if (!isPointInterpoQuery(pQuery)) { + return false; + } + + /* + * for interpolate point query, points that are directly before/after the specified point are required + */ + if (isFirstLastRowQuery(pQuery)) { + assert(!QUERY_IS_ASC_QUERY(pQuery)); + } else { + assert(QUERY_IS_ASC_QUERY(pQuery)); + } + assert(pPointInterpSupporter != NULL && pQuery->skey == pQuery->ekey); + + SCacheBlock *pBlock = NULL; + + qTrace("QInfo:%p get next data point, fileId:%d, slot:%d, pos:%d", GET_QINFO_ADDR(pQuery), pQuery->fileId, + pQuery->slot, pQuery->pos); + + // save the point that is directly after or equals to the specified point + getOneRowFromDataBlock(pRuntimeEnv, pPointInterpSupporter->pNextPoint, pQuery->pos); + + /* + * 1. for last_row query, return immediately. + * 2. the specified timestamp equals to the required key, interpolation according to neighbor points is not necessary + * for interp query. + */ + TSKEY actualKey = *(TSKEY *)pPointInterpSupporter->pNextPoint[0]; + if (isFirstLastRowQuery(pQuery) || actualKey == pQuery->skey) { + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); + + /* + * the retrieved ts may not equals to pMeterObj->lastKey due to cache re-allocation + * set the pQuery->ekey/pQuery->skey/pQuery->lastKey to be the new value. + */ + if (pQuery->ekey != actualKey) { + pQuery->skey = actualKey; + pQuery->ekey = actualKey; + pQuery->lastKey = actualKey; + pSupporter->rawSKey = actualKey; + pSupporter->rawEKey = actualKey; + } + return true; + } + + /* the qualified point is not the first point in data block */ + if (pQuery->pos > 0) { + int32_t prevPos = pQuery->pos - 1; + + /* save the point that is directly after the specified point */ + getOneRowFromDataBlock(pRuntimeEnv, pPointInterpSupporter->pPrevPoint, prevPos); + } else { + __block_search_fn_t searchFn = vnodeSearchKeyFunc[pMeterObj->searchAlgorithm]; + +// savePointPosition(&pRuntimeEnv->startPos, pQuery->fileId, pQuery->slot, pQuery->pos); + + // backwards movement would not set the pQuery->pos correct. We need to set it manually later. + moveToNextBlock(pRuntimeEnv, QUERY_DESC_FORWARD_STEP, searchFn, true); + + /* + * no previous data exists. + * reset the status and load the data block that contains the qualified point + */ + if (Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK)) { + dTrace("QInfo:%p no previous data block, start fileId:%d, slot:%d, pos:%d, qrange:%" PRId64 "-%" PRId64 + ", out of range", + GET_QINFO_ADDR(pQuery), pRuntimeEnv->startPos.fileId, pRuntimeEnv->startPos.slot, + pRuntimeEnv->startPos.pos, pQuery->skey, pQuery->ekey); + + // no result, return immediately + setQueryStatus(pQuery, QUERY_COMPLETED); + return false; + } else { // prev has been located + if (pQuery->fileId >= 0) { + pQuery->pos = pQuery->pBlock[pQuery->slot].numOfPoints - 1; + getOneRowFromDataBlock(pRuntimeEnv, pPointInterpSupporter->pPrevPoint, pQuery->pos); + + qTrace("QInfo:%p get prev data point, fileId:%d, slot:%d, pos:%d, pQuery->pos:%d", GET_QINFO_ADDR(pQuery), + pQuery->fileId, pQuery->slot, pQuery->pos, pQuery->pos); + } else { + // moveToNextBlock make sure there is a available cache block, if exists + assert(vnodeIsDatablockLoaded(pRuntimeEnv, pMeterObj, -1, true) == DISK_BLOCK_NO_NEED_TO_LOAD); + pBlock = &pRuntimeEnv->cacheBlock; + + pQuery->pos = pBlock->numOfPoints - 1; + getOneRowFromDataBlock(pRuntimeEnv, pPointInterpSupporter->pPrevPoint, pQuery->pos); + + qTrace("QInfo:%p get prev data point, fileId:%d, slot:%d, pos:%d, pQuery->pos:%d", GET_QINFO_ADDR(pQuery), + pQuery->fileId, pQuery->slot, pBlock->numOfPoints - 1, pQuery->pos); + } + } + } + + pQuery->skey = *(TSKEY *)pPointInterpSupporter->pPrevPoint[0]; + pQuery->ekey = *(TSKEY *)pPointInterpSupporter->pNextPoint[0]; + pQuery->lastKey = pQuery->skey; +#endif + return true; +} + +bool vnodeDoFilterData(SQuery* pQuery, int32_t elemPos) { + for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { + SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; + char* pElem = pFilterInfo->pData + pFilterInfo->info.info.bytes * elemPos; + + if(isNull(pElem, pFilterInfo->info.info.type)) { + return false; + } + + int32_t num = pFilterInfo->numOfFilters; + bool qualified = false; + for(int32_t j = 0; j < num; ++j) { + SColumnFilterElem* pFilterElem = &pFilterInfo->pFilters[j]; + if (pFilterElem->fp(pFilterElem, pElem, pElem)) { + qualified = true; + break; + } + } + + if (!qualified) { + return false; + } + } + + return true; +} + +bool vnodeFilterData(SQuery* pQuery, int32_t* numOfActualRead, int32_t index) { + (*numOfActualRead)++; + if (!vnodeDoFilterData(pQuery, index)) { + return false; + } + + if (pQuery->limit.offset > 0) { + pQuery->limit.offset--; // ignore this qualified row + return false; + } + + return true; +} + +int64_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; + bool hasMainFunction = hasMainOutput(pQuery); + + int64_t maxOutput = 0; + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; + + /* + * ts, tag, tagprj function can not decide the output number of current query + * the number of output result is decided by main output + */ + if (hasMainFunction && + (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ)) { + continue; + } + + SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + if (pResInfo != NULL && maxOutput < pResInfo->numOfRes) { + maxOutput = pResInfo->numOfRes; + } + } + + return maxOutput; +} + +static int32_t getGroupResultId(int32_t groupIndex) { + int32_t base = 200000; + return base + (groupIndex * 10000); +} + +bool isGroupbyNormalCol(SSqlGroupbyExpr *pGroupbyExpr) { + if (pGroupbyExpr == NULL || pGroupbyExpr->numOfGroupCols == 0) { + return false; + } + + for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { + SColIndexEx *pColIndex = &pGroupbyExpr->columnInfo[i]; + if (pColIndex->flag == TSDB_COL_NORMAL) { + /* + * make sure the normal column locates at the second position if tbname exists in group by clause + */ + if (pGroupbyExpr->numOfGroupCols > 1) { + assert(pColIndex->colIdx > 0); + } + + return true; + } + } + + return false; +} + +int16_t getGroupbyColumnType(SQuery *pQuery, SSqlGroupbyExpr *pGroupbyExpr) { + assert(pGroupbyExpr != NULL); + + int32_t colId = -2; + int16_t type = TSDB_DATA_TYPE_NULL; + + for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { + SColIndexEx *pColIndex = &pGroupbyExpr->columnInfo[i]; + if (pColIndex->flag == TSDB_COL_NORMAL) { + colId = pColIndex->colId; + break; + } + } + + for (int32_t i = 0; i < pQuery->numOfCols; ++i) { + if (colId == pQuery->colList[i].info.colId) { + type = pQuery->colList[i].info.type; + break; + } + } + + return type; +} + +bool isSelectivityWithTagsQuery(SQuery *pQuery) { + bool hasTags = false; + int32_t numOfSelectivity = 0; + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functId = pQuery->pSelectExpr[i].pBase.functionId; + if (functId == TSDB_FUNC_TAG_DUMMY || functId == TSDB_FUNC_TS_DUMMY) { + hasTags = true; + continue; + } + + if ((aAggs[functId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { + numOfSelectivity++; + } + } + + if (numOfSelectivity > 0 && hasTags) { + return true; + } + + return false; +} + +bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].pBase.functionId == TSDB_FUNC_TS_COMP; } + +bool doRevisedResultsByLimit(SQInfo *pQInfo) { + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + if ((pQuery->limit.limit > 0) && (pQuery->rec.pointsRead + pQInfo->rec.pointsRead > pQuery->limit.limit)) { + pQuery->rec.pointsRead = pQuery->limit.limit - pQInfo->rec.pointsRead; + + // query completed + setQueryStatus(pQuery, QUERY_COMPLETED); + return true; + } + + return false; +} + +/** + * + * @param pQuery + * @param pDataBlockInfo + * @param forwardStep + * @return TRUE means query not completed, FALSE means query is completed + */ +static bool queryPaused(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo, int32_t forwardStep) { + // output buffer is full, pause current query + if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { +// assert((QUERY_IS_ASC_QUERY(pQuery) && forwardStep + pQuery->pos <= pDataBlockInfo->size) || +// (!QUERY_IS_ASC_QUERY(pQuery) && pQuery->pos - forwardStep + 1 >= 0)); +// + return true; + } + + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { + return true; + } + + return false; +} + +static bool isTopBottomQuery(SQuery *pQuery) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + if (functionId == TSDB_FUNC_TS) { + continue; + } + + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { + return true; + } + } + + return false; +} + +static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, int32_t columnIndex) { + // no SField info exist, or column index larger than the output column, no result. + if (pStatis == NULL) { + return NULL; + } + + // for a tag column, no corresponding field info + SColIndexEx *pColIndexEx = &pQuery->pSelectExpr[columnIndex].pBase.colInfo; + if (TSDB_COL_IS_TAG(pColIndexEx->flag)) { + return NULL; + } + + /* + * Choose the right column field info by field id, since the file block may be out of date, + * which means the newest table schema is not equalled to the schema of this block. + */ + for (int32_t i = 0; i < pDataBlockInfo->numOfCols; ++i) { + if (pColIndexEx->colId == pStatis[i].colId) { + return &pStatis[i]; + } + } + + return NULL; +} + +static bool hasNullValue(SQuery *pQuery, int32_t col, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, + SDataStatis **pColStatis) { + if (TSDB_COL_IS_TAG(pQuery->pSelectExpr[col].pBase.colInfo.flag) || pStatis == NULL) { + return false; + } + + *pColStatis = getStatisInfo(pQuery, pStatis, pDataBlockInfo, col); + if ((*pColStatis) != NULL && (*pColStatis)->numOfNull == 0) { + return false; + } + + return true; +} + +static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, char *pData, + int16_t bytes) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + int32_t *p1 = (int32_t *)taosHashGet(pWindowResInfo->hashList, pData, bytes); + if (p1 != NULL) { + pWindowResInfo->curIndex = *p1; + } else { // more than the capacity, reallocate the resources + if (pWindowResInfo->size >= pWindowResInfo->capacity) { + int64_t newCap = pWindowResInfo->capacity * 2; + + char *t = realloc(pWindowResInfo->pResult, newCap * sizeof(SWindowResult)); + if (t != NULL) { + pWindowResInfo->pResult = (SWindowResult *)t; + memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, sizeof(SWindowResult) * pWindowResInfo->capacity); + } else { + // todo + } + + for (int32_t i = pWindowResInfo->capacity; i < newCap; ++i) { + SPosInfo pos = {-1, -1}; + createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, &pos); + } + + pWindowResInfo->capacity = newCap; + } + + // add a new result set for a new group + pWindowResInfo->curIndex = pWindowResInfo->size++; + taosHashPut(pWindowResInfo->hashList, pData, bytes, (char *)&pWindowResInfo->curIndex, sizeof(int32_t)); + } + + return getWindowResult(pWindowResInfo, pWindowResInfo->curIndex); +} + +// get the correct time window according to the handled timestamp +static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t ts, SQuery *pQuery) { + STimeWindow w = {0}; + + if (pWindowResInfo->curIndex == -1) { // the first window, from the previous stored value + w.skey = pWindowResInfo->prevSKey; + w.ekey = w.skey + pQuery->intervalTime - 1; + } else { + int32_t slot = curTimeWindow(pWindowResInfo); + w = getWindowResult(pWindowResInfo, slot)->window; + } + + if (w.skey > ts || w.ekey < ts) { + int64_t st = w.skey; + + if (st > ts) { + st -= ((st - ts + pQuery->slidingTime - 1) / pQuery->slidingTime) * pQuery->slidingTime; + } + + int64_t et = st + pQuery->intervalTime - 1; + if (et < ts) { + st += ((ts - et + pQuery->slidingTime - 1) / pQuery->slidingTime) * pQuery->slidingTime; + } + + w.skey = st; + w.ekey = w.skey + pQuery->intervalTime - 1; + } + + /* + * query border check, skey should not be bounded by the query time range, since the value skey will + * be used as the time window index value. So we only change ekey of time window accordingly. + */ + if (w.ekey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) { + w.ekey = pQuery->window.ekey; + } + + assert(ts >= w.skey && ts <= w.ekey && w.skey != 0); + + return w; +} + +static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t sid, + int32_t numOfRowsPerPage) { + if (pWindowRes->pos.pageId != -1) { + return 0; + } + + tFilePage *pData = NULL; + + // in the first scan, new space needed for results + int32_t pageId = -1; + SIDList list = getDataBufPagesIdList(pResultBuf, sid); + + if (list.size == 0) { + pData = getNewDataBuf(pResultBuf, sid, &pageId); + } else { + pageId = getLastPageId(&list); + pData = getResultBufferPageById(pResultBuf, pageId); + + if (pData->numOfElems >= numOfRowsPerPage) { + pData = getNewDataBuf(pResultBuf, sid, &pageId); + if (pData != NULL) { + assert(pData->numOfElems == 0); // number of elements must be 0 for new allocated buffer + } + } + } + + if (pData == NULL) { + return -1; + } + + // set the number of rows in current disk page + if (pWindowRes->pos.pageId == -1) { // not allocated yet, allocate new buffer + pWindowRes->pos.pageId = pageId; + pWindowRes->pos.rowId = pData->numOfElems++; + } + + return 0; +} + +static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, int32_t sid, + STimeWindow *win) { + assert(win->skey <= win->ekey); + SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; + + SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&win->skey, TSDB_KEYSIZE); + if (pWindowRes == NULL) { + return -1; + } + + // not assign result buffer yet, add new result buffer + if (pWindowRes->pos.pageId == -1) { + int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, sid, pRuntimeEnv->numOfRowsPerPage); + if (ret != 0) { + return -1; + } + } + + // set time window for current result + pWindowRes->window = *win; + + setWindowResOutputBuf(pRuntimeEnv, pWindowRes); + initCtxOutputBuf(pRuntimeEnv); + + return TSDB_CODE_SUCCESS; +} + +static SWindowStatus *getTimeWindowResStatus(SWindowResInfo *pWindowResInfo, int32_t slot) { + assert(slot >= 0 && slot < pWindowResInfo->size); + return &pWindowResInfo->pResult[slot].status; +} + +static int32_t getForwardStepsInBlock(int32_t numOfPoints, __block_search_fn_t searchFn, TSKEY ekey, int16_t pos, + int16_t order, int64_t *pData) { + int32_t endPos = searchFn((char *)pData, numOfPoints, ekey, order); + int32_t forwardStep = 0; + + if (endPos >= 0) { + forwardStep = (order == TSQL_SO_ASC) ? (endPos - pos) : (pos - endPos); + assert(forwardStep >= 0); + + // endPos data is equalled to the key so, we do need to read the element in endPos + if (pData[endPos] == ekey) { + forwardStep += 1; + } + } + + return forwardStep; +} + +/** + * NOTE: the query status only set for the first scan of master scan. + */ +static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, SWindowResInfo *pWindowResInfo) { + SQuery *pQuery = pRuntimeEnv->pQuery; + if (pRuntimeEnv->scanFlag != MASTER_SCAN || (!isIntervalQuery(pQuery))) { + return; + } + + // no qualified results exist, abort check + if (pWindowResInfo->size == 0) { + return; + } + + // query completed + if ((lastKey >= pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (lastKey <= pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) { + closeAllTimeWindow(pWindowResInfo); + + pWindowResInfo->curIndex = pWindowResInfo->size - 1; + setQueryStatus(pQuery, QUERY_COMPLETED | QUERY_RESBUF_FULL); + } else { // set the current index to be the last unclosed window + int32_t i = 0; + int64_t skey = 0; + + for (i = 0; i < pWindowResInfo->size; ++i) { + SWindowResult *pResult = &pWindowResInfo->pResult[i]; + if (pResult->status.closed) { + continue; + } + + if ((pResult->window.ekey <= lastKey && QUERY_IS_ASC_QUERY(pQuery)) || + (pResult->window.skey >= lastKey && !QUERY_IS_ASC_QUERY(pQuery))) { + closeTimeWindow(pWindowResInfo, i); + } else { + skey = pResult->window.skey; + break; + } + } + + // all windows are closed, set the last one to be the skey + if (skey == 0) { + assert(i == pWindowResInfo->size); + pWindowResInfo->curIndex = pWindowResInfo->size - 1; + } else { + pWindowResInfo->curIndex = i; + } + + pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex].window.skey; + + // the number of completed slots are larger than the threshold, dump to client immediately. + int32_t n = numOfClosedTimeWindow(pWindowResInfo); + if (n > pWindowResInfo->threshold) { + setQueryStatus(pQuery, QUERY_RESBUF_FULL); + } + + qTrace("QInfo:%p total window:%d, closed:%d", GET_QINFO_ADDR(pQuery), pWindowResInfo->size, n); + } + + assert(pWindowResInfo->prevSKey != 0); +} + +static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn, + int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, bool updateLastKey) { + assert(startPos >= 0 && startPos < pDataBlockInfo->size); + + int32_t num = -1; + int32_t order = pQuery->order.order; + + int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); + + if (QUERY_IS_ASC_QUERY(pQuery)) { + if (ekey < pDataBlockInfo->window.ekey) { + num = getForwardStepsInBlock(pDataBlockInfo->size, searchFn, ekey, startPos, order, pPrimaryColumn); + if (num == 0) { // no qualified data in current block, do not update the lastKey value + assert(ekey < pPrimaryColumn[startPos]); + } else { + if (updateLastKey) { + pQuery->lastKey = pPrimaryColumn[startPos + (num - 1)] + step; + } + } + } else { + num = pDataBlockInfo->size - startPos; + if (updateLastKey) { + pQuery->lastKey = pDataBlockInfo->window.ekey + step; + } + } + } else { // desc + if (ekey > pDataBlockInfo->window.skey) { + num = getForwardStepsInBlock(pDataBlockInfo->size, searchFn, ekey, startPos, order, pPrimaryColumn); + if (num == 0) { // no qualified data in current block, do not update the lastKey value + assert(ekey > pPrimaryColumn[startPos]); + } else { + if (updateLastKey) { + pQuery->lastKey = pPrimaryColumn[startPos - (num - 1)] + step; + } + } + } else { + num = startPos + 1; + if (updateLastKey) { + pQuery->lastKey = pDataBlockInfo->window.skey + step; + } + } + } + + assert(num >= 0); + return num; +} + +static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus *pStatus, STimeWindow *pWin, + int32_t startPos, int32_t forwardStep, TSKEY* tsBuf) { + SQuery * pQuery = pRuntimeEnv->pQuery; + SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; + + if (IS_MASTER_SCAN(pRuntimeEnv) || pStatus->closed) { + for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + + pCtx[k].nStartQueryTimestamp = pWin->skey; + pCtx[k].size = forwardStep; + pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? startPos : startPos - (forwardStep - 1); + + if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { + pCtx[k].ptsList = &tsBuf[pCtx[k].startOffset]; + } + + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { + aAggs[functionId].xFunction(&pCtx[k]); + } + } + } +} + +static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus *pStatus, STimeWindow *pWin, + int32_t offset) { + SQuery * pQuery = pRuntimeEnv->pQuery; + SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; + + if (IS_MASTER_SCAN(pRuntimeEnv) || pStatus->closed) { + for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + pCtx[k].nStartQueryTimestamp = pWin->skey; + + int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { + aAggs[functionId].xFunctionF(&pCtx[k], offset); + } + } + } +} + +static int32_t getNextQualifiedWindow(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow *pNextWin, + SWindowResInfo *pWindowResInfo, SDataBlockInfo *pDataBlockInfo, + TSKEY *primaryKeys, __block_search_fn_t searchFn) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + while (1) { + if ((pNextWin->ekey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (pNextWin->skey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) { + return -1; + } + + getNextTimeWindow(pQuery, pNextWin); + + // next time window is not in current block + if ((pNextWin->skey > pDataBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (pNextWin->ekey < pDataBlockInfo->window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { + return -1; + } + + TSKEY startKey = -1; + if (QUERY_IS_ASC_QUERY(pQuery)) { + startKey = pNextWin->skey; + if (startKey < pQuery->window.skey) { + startKey = pQuery->window.skey; + } + } else { + startKey = pNextWin->ekey; + if (startKey > pQuery->window.skey) { + startKey = pQuery->window.skey; + } + } + + int32_t startPos = searchFn((char *)primaryKeys, pDataBlockInfo->size, startKey, pQuery->order.order); + + /* + * This time window does not cover any data, try next time window, + * this case may happen when the time window is too small + */ + if ((primaryKeys[startPos] > pNextWin->ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (primaryKeys[startPos] < pNextWin->skey && !QUERY_IS_ASC_QUERY(pQuery))) { + continue; + } + + return startPos; + } +} + +static TSKEY reviseWindowEkey(SQuery *pQuery, STimeWindow *pWindow) { + TSKEY ekey = -1; + if (QUERY_IS_ASC_QUERY(pQuery)) { + ekey = pWindow->ekey; + if (ekey > pQuery->window.ekey) { + ekey = pQuery->window.ekey; + } + } else { + ekey = pWindow->skey; + if (ekey < pQuery->window.ekey) { + ekey = pQuery->window.ekey; + } + } + + return ekey; +} + +char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int32_t col, int32_t size, + SArray *pDataBlock) { + SQuery * pQuery = pRuntimeEnv->pQuery; + SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; + + char *dataBlock = NULL; + + int32_t functionId = pQuery->pSelectExpr[col].pBase.functionId; + + if (functionId == TSDB_FUNC_ARITHM) { + sas->pExpr = &pQuery->pSelectExpr[col]; + + // set the start offset to be the lowest start position, no matter asc/desc query order + if (QUERY_IS_ASC_QUERY(pQuery)) { + pCtx->startOffset = pQuery->pos; + } else { + pCtx->startOffset = pQuery->pos - (size - 1); + } + + for (int32_t i = 0; i < pQuery->numOfCols; ++i) { + SColumnInfo *pColMsg = &pQuery->colList[i].info; + assert(0); +// char * pData = doGetDataBlocks(pQuery, pRuntimeEnv->colDataBuffer, pQuery->colList[i].colIdxInBuf); + + sas->elemSize[i] = pColMsg->bytes; +// sas->data[i] = pData + pCtx->startOffset * sas->elemSize[i]; // start from the offset + } + + sas->numOfCols = pQuery->numOfCols; + sas->offset = 0; + } else { // other type of query function + SColIndexEx *pCol = &pQuery->pSelectExpr[col].pBase.colInfo; + if (TSDB_COL_IS_TAG(pCol->flag)) { + dataBlock = NULL; + } else { + /* + * the colIdx is acquired from the first meter of all qualified meters in this vnode during query prepare stage, + * the remain meter may not have the required column in cache actually. + * So, the validation of required column in cache with the corresponding meter schema is reinforced. + */ + + if (pDataBlock == NULL) { + return NULL; + } + + int32_t numOfCols = taosArrayGetSize(pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoEx *p = taosArrayGet(pDataBlock, i); + if (pCol->colId == p->info.colId) { + dataBlock = p->pData; + break; + } + } + } + } + + return dataBlock; +} + +/** + * + * @param pRuntimeEnv + * @param forwardStep + * @param primaryKeyCol + * @param pFields + * @param isDiskFileBlock + * @return the incremental number of output value, so it maybe 0 for fixed number of query, + * such as count/min/max etc. + */ +static int32_t blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, + SDataBlockInfo *pDataBlockInfo, SWindowResInfo *pWindowResInfo, + __block_search_fn_t searchFn, SArray *pDataBlock) { + SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; + SQuery * pQuery = pRuntimeEnv->pQuery; + + SColumnInfoEx *pColInfo = NULL; + TSKEY * primaryKeyCol = NULL; + + if (pDataBlock != NULL) { + pColInfo = taosArrayGet(pDataBlock, 0); + primaryKeyCol = (TSKEY *)(pColInfo->pData); + } + + pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0:pDataBlockInfo->size - 1; + int64_t prevNumOfRes = getNumOfResult(pRuntimeEnv); + + SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutputCols, sizeof(SArithmeticSupport)); + + for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + + SDataStatis *tpField = NULL; + bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &tpField); + char * dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->size, pDataBlock); + + setExecParams(pQuery, &pCtx[k], dataBlock, (char *)primaryKeyCol, pDataBlockInfo->size, functionId, tpField, + hasNull, &sasArray[k], pRuntimeEnv->scanFlag); + } + + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + if (isIntervalQuery(pQuery)) { + int32_t offset = GET_COL_DATA_POS(pQuery, 0, step); + TSKEY ts = primaryKeyCol[offset]; + + STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); + assert(0); +// if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pRuntimeEnv->pTabObj->sid, &win) != TSDB_CODE_SUCCESS) { +// return 0; +// } + + TSKEY ekey = reviseWindowEkey(pQuery, &win); + int32_t forwardStep = + getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, primaryKeyCol, pQuery->pos, ekey, searchFn, true); + + SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); + doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, pQuery->pos, forwardStep, primaryKeyCol); + + int32_t index = pWindowResInfo->curIndex; + STimeWindow nextWin = win; + + while (1) { + int32_t startPos = + getNextQualifiedWindow(pRuntimeEnv, &nextWin, pWindowResInfo, pDataBlockInfo, primaryKeyCol, searchFn); + if (startPos < 0) { + break; + } + + // null data, failed to allocate more memory buffer +// int32_t sid = pRuntimeEnv->pTabObj->sid; + int32_t sid = 0; + assert(0); + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, sid, &nextWin) != TSDB_CODE_SUCCESS) { + break; + } + + ekey = reviseWindowEkey(pQuery, &nextWin); + forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, primaryKeyCol, startPos, ekey, searchFn, true); + + pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); + + doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, startPos, forwardStep, primaryKeyCol); + } + + pWindowResInfo->curIndex = index; + } else { + /* + * the sqlfunctionCtx parameters should be set done before all functions are invoked, + * since the selectivity + tag_prj query needs all parameters been set done. + * tag_prj function are changed to be TSDB_FUNC_TAG_DUMMY + */ + for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { + aAggs[functionId].xFunction(&pCtx[k]); + } + } + } + + /* + * No need to calculate the number of output results for group-by normal columns, interval query + * because the results of group by normal column is put into intermediate buffer. + */ + int32_t num = 0; + if (!isIntervalQuery(pQuery)) { + num = getNumOfResult(pRuntimeEnv) - prevNumOfRes; + } + + tfree(sasArray); + return (int32_t)num; +} + +static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pData, int16_t type, int16_t bytes) { + if (isNull(pData, type)) { // ignore the null value + return -1; + } + + int32_t GROUPRESULTID = 1; + + SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; + + SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pData, bytes); + if (pWindowRes == NULL) { + return -1; + } + + // not assign result buffer yet, add new result buffer + if (pWindowRes->pos.pageId == -1) { + int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage); + if (ret != 0) { + return -1; + } + } + + setWindowResOutputBuf(pRuntimeEnv, pWindowRes); + initCtxOutputBuf(pRuntimeEnv); + return TSDB_CODE_SUCCESS; +} + +static char *getGroupbyColumnData(SQuery *pQuery, SData **data, int16_t *type, int16_t *bytes) { + char *groupbyColumnData = NULL; + + SSqlGroupbyExpr *pGroupbyExpr = pQuery->pGroupbyExpr; + + for (int32_t k = 0; k < pGroupbyExpr->numOfGroupCols; ++k) { + if (pGroupbyExpr->columnInfo[k].flag == TSDB_COL_TAG) { + continue; + } + + int16_t colIndex = -1; + int32_t colId = pGroupbyExpr->columnInfo[k].colId; + + for (int32_t i = 0; i < pQuery->numOfCols; ++i) { + if (pQuery->colList[i].info.colId == colId) { + colIndex = i; + break; + } + } + + assert(colIndex >= 0 && colIndex < pQuery->numOfCols); + + *type = pQuery->colList[colIndex].info.type; + *bytes = pQuery->colList[colIndex].info.bytes; + +// groupbyColumnData = doGetDataBlocks(pQuery, data, pQuery->colList[colIndex].inf); + break; + } + + return groupbyColumnData; +} + +static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + STSElem elem = tsBufGetElem(pRuntimeEnv->pTSBuf); + SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; + + // compare tag first + if (pCtx[0].tag.i64Key != elem.tag) { + return TS_JOIN_TAG_NOT_EQUALS; + } + + TSKEY key = *(TSKEY *)(pCtx[0].aInputElemBuf + TSDB_KEYSIZE * offset); + +#if defined(_DEBUG_VIEW) + printf("elem in comp ts file:%" PRId64 ", key:%" PRId64 + ", tag:%d, id:%s, query order:%d, ts order:%d, traverse:%d, index:%d\n", + elem.ts, key, elem.tag, pRuntimeEnv->pTabObj->meterId, pQuery->order.order, pRuntimeEnv->pTSBuf->tsOrder, + pRuntimeEnv->pTSBuf->cur.order, pRuntimeEnv->pTSBuf->cur.tsIndex); +#endif + + if (QUERY_IS_ASC_QUERY(pQuery)) { + if (key < elem.ts) { + return TS_JOIN_TS_NOT_EQUALS; + } else if (key > elem.ts) { + assert(false); + } + } else { + if (key > elem.ts) { + return TS_JOIN_TS_NOT_EQUALS; + } else if (key < elem.ts) { + assert(false); + } + } + + return TS_JOIN_TS_EQUAL; +} + +static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + + if (pResInfo->complete || functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { + return false; + } + + // in the supplementary scan, only the following functions need to be executed + if (IS_SUPPLEMENT_SCAN(pRuntimeEnv) && + !(functionId == TSDB_FUNC_LAST_DST || functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_FIRST || + functionId == TSDB_FUNC_LAST || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TS)) { + return false; + } + + return true; +} + +static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis* pStatis, + SDataBlockInfo *pDataBlockInfo, SWindowResInfo *pWindowResInfo, SArray* pDataBlock) { + SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; + SQuery * pQuery = pRuntimeEnv->pQuery; + TSKEY * primaryKeyCol = (TSKEY *)taosArrayGet(pDataBlock, 0); + +// SData **data = pRuntimeEnv->colDataBuffer; + + int64_t prevNumOfRes = 0; + bool groupbyStateValue = isGroupbyNormalCol(pQuery->pGroupbyExpr); + + if (!groupbyStateValue) { + prevNumOfRes = getNumOfResult(pRuntimeEnv); + } + + SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutputCols, sizeof(SArithmeticSupport)); + + int16_t type = 0; + int16_t bytes = 0; + + char *groupbyColumnData = NULL; + if (groupbyStateValue) { + assert(0); +// groupbyColumnData = getGroupbyColumnData(pQuery, data, &type, &bytes); + } + + for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + + SDataStatis* pColStatis = NULL; + + bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &pColStatis); + char *dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->size, pDataBlock); + + setExecParams(pQuery, &pCtx[k], dataBlock, (char *)primaryKeyCol, pDataBlockInfo->size, functionId, pColStatis, hasNull, + &sasArray[k], pRuntimeEnv->scanFlag); + } + + // set the input column data + for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { + SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; + assert(0); + /* + * NOTE: here the tbname/tags column cannot reach here, since it will never be a filter column, + * so we do NOT check if is a tag or not + */ +// pFilterInfo->pData = doGetDataBlocks(pQuery, data, pFilterInfo->info.colIdxInBuf); + } + + int32_t numOfRes = 0; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + + // from top to bottom in desc + // from bottom to top in asc order + if (pRuntimeEnv->pTSBuf != NULL) { + SQInfo *pQInfo = (SQInfo *)GET_QINFO_ADDR(pQuery); + qTrace("QInfo:%p process data rows, numOfRows:%d, query order:%d, ts comp order:%d", pQInfo, pDataBlockInfo->size, + pQuery->order.order, pRuntimeEnv->pTSBuf->cur.order); + } + + int32_t j = 0; + TSKEY lastKey = -1; + + for (j = 0; j < pDataBlockInfo->size; ++j) { + int32_t offset = GET_COL_DATA_POS(pQuery, j, step); + + if (pRuntimeEnv->pTSBuf != NULL) { + int32_t r = doTSJoinFilter(pRuntimeEnv, offset); + if (r == TS_JOIN_TAG_NOT_EQUALS) { + break; + } else if (r == TS_JOIN_TS_NOT_EQUALS) { + continue; + } else { + assert(r == TS_JOIN_TS_EQUAL); + } + } + + if (pQuery->numOfFilterCols > 0 && (!vnodeDoFilterData(pQuery, offset))) { + continue; + } + + // interval window query + if (isIntervalQuery(pQuery)) { + // decide the time window according to the primary timestamp + int64_t ts = primaryKeyCol[offset]; + STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); + + assert(0); + int32_t ret = 0; +// int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pRuntimeEnv->pTabObj->sid, &win); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + continue; + } + + // all startOffset are identical + offset -= pCtx[0].startOffset; + + SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); + doRowwiseApplyFunctions(pRuntimeEnv, pStatus, &win, offset); + + lastKey = ts; + STimeWindow nextWin = win; + int32_t index = pWindowResInfo->curIndex; + assert(0); + int32_t sid = 0;//pRuntimeEnv->pTabObj->sid; + + while (1) { + getNextTimeWindow(pQuery, &nextWin); + if (pWindowResInfo->startTime > nextWin.skey || (nextWin.skey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (nextWin.skey > pQuery->window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { + break; + } + + if (ts < nextWin.skey || ts > nextWin.ekey) { + break; + } + + // null data, failed to allocate more memory buffer + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, sid, &nextWin) != TSDB_CODE_SUCCESS) { + break; + } + + pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); + doRowwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, offset); + } + + pWindowResInfo->curIndex = index; + } else { // other queries + // decide which group this rows belongs to according to current state value + if (groupbyStateValue) { + char *stateVal = groupbyColumnData + bytes * offset; + + int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, stateVal, type, bytes); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + continue; + } + } + + // update the lastKey + lastKey = primaryKeyCol[offset]; + + // all startOffset are identical + offset -= pCtx[0].startOffset; + + for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { + aAggs[functionId].xFunctionF(&pCtx[k], offset); + } + } + } + + if (pRuntimeEnv->pTSBuf != NULL) { + // if timestamp filter list is empty, quit current query + if (!tsBufNextPos(pRuntimeEnv->pTSBuf)) { + setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); + break; + } + } + + /* + * pointsOffset is the maximum available space in result buffer update the actual forward step for query that + * requires checking buffer during loop + */ + if ((pQuery->checkBufferInLoop == 1) && (++numOfRes) >= pQuery->pointsOffset) { + pQuery->lastKey = lastKey + step; + assert(0); +// *forwardStep = j + 1; + break; + } + } + + free(sasArray); + + /* + * No need to calculate the number of output results for group-by normal columns, interval query + * because the results of group by normal column is put into intermediate buffer. + */ + int32_t num = 0; + if (!groupbyStateValue && !isIntervalQuery(pQuery)) { + num = getNumOfResult(pRuntimeEnv) - prevNumOfRes; + } + + return num; +} + +static int32_t reviseForwardSteps(SQueryRuntimeEnv *pRuntimeEnv, int32_t forwardStep) { + /* + * 1. If value filter exists, we try all data in current block, and do not set the QUERY_RESBUF_FULL flag. + * + * 2. In case of top/bottom/ts_comp query, the checkBufferInLoop == 1 and pQuery->numOfFilterCols + * may be 0 or not. We do not check the capacity of output buffer, since the filter function will do it. + * + * 3. In handling the query of secondary query of join, tsBuf servers as a ts filter. + */ + SQuery *pQuery = pRuntimeEnv->pQuery; + + if (isTopBottomQuery(pQuery) || isTSCompQuery(pQuery) || pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) { + return forwardStep; + } + + // current buffer does not have enough space, try in the next loop + if ((pQuery->checkBufferInLoop == 1) && (pQuery->pointsOffset <= forwardStep)) { + forwardStep = pQuery->pointsOffset; + } + + return forwardStep; +} + +static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, + SDataStatis *pStatis, __block_search_fn_t searchFn, int32_t *numOfRes, + SWindowResInfo *pWindowResInfo, SArray *pDataBlock) { + SQuery *pQuery = pRuntimeEnv->pQuery; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + *numOfRes = rowwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); + } else { + *numOfRes = blockwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); + } + + TSKEY lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pDataBlockInfo->window.ekey : pDataBlockInfo->window.skey; + pQuery->lastKey = lastKey + step; + + doCheckQueryCompleted(pRuntimeEnv, lastKey, pWindowResInfo); + + // interval query with limit applied + if (isIntervalQuery(pQuery) && pQuery->limit.limit > 0 && + (pQuery->limit.limit + pQuery->limit.offset) <= numOfClosedTimeWindow(pWindowResInfo) && + pRuntimeEnv->scanFlag == MASTER_SCAN) { + setQueryStatus(pQuery, QUERY_COMPLETED); + } + + assert(*numOfRes >= 0); + + // check if buffer is large enough for accommodating all qualified points + if (*numOfRes > 0 && pQuery->checkBufferInLoop == 1) { + pQuery->pointsOffset -= *numOfRes; + if (pQuery->pointsOffset <= 0) { // todo return correct numOfRes for ts_comp function + pQuery->pointsOffset = 0; + setQueryStatus(pQuery, QUERY_RESBUF_FULL); + } + } + + return 0; +} + +void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, char *primaryColumnData, int32_t size, + int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t scanFlag) { + pCtx->scanFlag = scanFlag; + + pCtx->aInputElemBuf = inputData; + pCtx->hasNull = hasNull; + + if (pStatis != NULL) { + pCtx->preAggVals.isSet = true; + pCtx->preAggVals.size = size; + pCtx->preAggVals.statis = *pStatis; + } else { + pCtx->preAggVals.isSet = false; + } + + if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0 && (primaryColumnData != NULL)) { + pCtx->ptsList = (int64_t *)(primaryColumnData); + } + + if (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) { + // last_dist or first_dist function + // store the first&last timestamp into the intermediate buffer [1], the true + // value may be null but timestamp will never be null + pCtx->ptsList = (int64_t *)(primaryColumnData); + } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_TWA || + functionId == TSDB_FUNC_DIFF || (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE)) { + /* + * leastsquares function needs two columns of input, currently, the x value of linear equation is set to + * timestamp column, and the y-value is the column specified in pQuery->pSelectExpr[i].colIdxInBuffer + * + * top/bottom function needs timestamp to indicate when the + * top/bottom values emerge, so does diff function + */ + if (functionId == TSDB_FUNC_TWA) { + STwaInfo *pTWAInfo = GET_RES_INFO(pCtx)->interResultBuf; + pTWAInfo->SKey = pQuery->window.skey; + pTWAInfo->EKey = pQuery->window.ekey; + } + + pCtx->ptsList = (int64_t *)(primaryColumnData); + + } else if (functionId == TSDB_FUNC_ARITHM) { + pCtx->param[1].pz = param; + } + + pCtx->startOffset = 0; + pCtx->size = size; + +#if defined(_DEBUG_VIEW) + // int64_t *tsList = (int64_t *)primaryColumnData; +// int64_t s = tsList[0]; +// int64_t e = tsList[size - 1]; + +// if (IS_DATA_BLOCK_LOADED(blockStatus)) { +// dTrace("QInfo:%p query ts:%lld-%lld, offset:%d, rows:%d, bstatus:%d, +// functId:%d", GET_QINFO_ADDR(pQuery), +// s, e, startOffset, size, blockStatus, functionId); +// } else { +// dTrace("QInfo:%p block not loaded, bstatus:%d", +// GET_QINFO_ADDR(pQuery), blockStatus); +// } +#endif +} + +// set the output buffer for the selectivity + tag query +static void setCtxTagColumnInfo(SQuery *pQuery, SQLFunctionCtx *pCtx) { + if (isSelectivityWithTagsQuery(pQuery)) { + int32_t num = 0; + SQLFunctionCtx *p = NULL; + + int16_t tagLen = 0; + + SQLFunctionCtx **pTagCtx = calloc(pQuery->numOfOutputCols, POINTER_BYTES); + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SSqlFuncExprMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].pBase; + if (pSqlFuncMsg->functionId == TSDB_FUNC_TAG_DUMMY || pSqlFuncMsg->functionId == TSDB_FUNC_TS_DUMMY) { + tagLen += pCtx[i].outputBytes; + pTagCtx[num++] = &pCtx[i]; + } else if ((aAggs[pSqlFuncMsg->functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { + p = &pCtx[i]; + } else if (pSqlFuncMsg->functionId == TSDB_FUNC_TS || pSqlFuncMsg->functionId == TSDB_FUNC_TAG) { + // tag function may be the group by tag column + // ts may be the required primary timestamp column + continue; + } else { + // the column may be the normal column, group by normal_column, the functionId is TSDB_FUNC_PRJ + } + } + + p->tagInfo.pTagCtxList = pTagCtx; + p->tagInfo.numOfTagCols = num; + p->tagInfo.tagsLen = tagLen; + } +} + +static void setWindowResultInfo(SResultInfo *pResultInfo, SQuery *pQuery, bool isStableQuery) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + setResultInfoBuf(&pResultInfo[i], pQuery->pSelectExpr[i].interResBytes, isStableQuery); + } +} + +static int32_t setupQueryRuntimeEnv(void *pMeterObj, SQuery *pQuery, SQueryRuntimeEnv *pRuntimeEnv, + SColumnModel *pTagsSchema, int16_t order, bool isSTableQuery) { + dTrace("QInfo:%p setup runtime env", GET_QINFO_ADDR(pQuery)); + + pRuntimeEnv->pTabObj = pMeterObj; + pRuntimeEnv->pQuery = pQuery; + + pRuntimeEnv->resultInfo = calloc(pQuery->numOfOutputCols, sizeof(SResultInfo)); + pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutputCols, sizeof(SQLFunctionCtx)); + + if (pRuntimeEnv->resultInfo == NULL || pRuntimeEnv->pCtx == NULL) { + goto _error_clean; + } + + pRuntimeEnv->offset[0] = 0; + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SSqlFuncExprMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].pBase; + SColIndexEx * pColIndexEx = &pSqlFuncMsg->colInfo; + + SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; + + if (TSDB_COL_IS_TAG(pSqlFuncMsg->colInfo.flag)) { // process tag column info + SSchema *pSchema = getColumnModelSchema(pTagsSchema, pColIndexEx->colIdx); + + pCtx->inputType = pSchema->type; + pCtx->inputBytes = pSchema->bytes; + } else { + assert(0); +// pCtx->inputType = GET_COLUMN_TYPE(pQuery, i); +// pCtx->inputBytes = GET_COLUMN_BYTES(pQuery, i); + } + + pCtx->ptsOutputBuf = NULL; + + pCtx->outputBytes = pQuery->pSelectExpr[i].resBytes; + pCtx->outputType = pQuery->pSelectExpr[i].resType; + + pCtx->order = pQuery->order.order; + pCtx->functionId = pSqlFuncMsg->functionId; + + pCtx->numOfParams = pSqlFuncMsg->numOfParams; + for (int32_t j = 0; j < pCtx->numOfParams; ++j) { + int16_t type = pSqlFuncMsg->arg[j].argType; + int16_t bytes = pSqlFuncMsg->arg[j].argBytes; + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + tVariantCreateFromBinary(&pCtx->param[j], pSqlFuncMsg->arg->argValue.pz, bytes, type); + } else { + tVariantCreateFromBinary(&pCtx->param[j], (char *)&pSqlFuncMsg->arg[j].argValue.i64, bytes, type); + } + } + + // set the order information for top/bottom query + int32_t functionId = pCtx->functionId; + + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { + int32_t f = pQuery->pSelectExpr[0].pBase.functionId; + assert(f == TSDB_FUNC_TS || f == TSDB_FUNC_TS_DUMMY); + + pCtx->param[2].i64Key = order; + pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; + pCtx->param[3].i64Key = functionId; + pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT; + + pCtx->param[1].i64Key = pQuery->order.orderColId; + } + + if (i > 0) { + pRuntimeEnv->offset[i] = pRuntimeEnv->offset[i - 1] + pRuntimeEnv->pCtx[i - 1].outputBytes; + } + } + + // set the intermediate result output buffer + setWindowResultInfo(pRuntimeEnv->resultInfo, pQuery, isSTableQuery); + + // if it is group by normal column, do not set output buffer, the output buffer is pResult + if (!isGroupbyNormalCol(pQuery->pGroupbyExpr) && !isSTableQuery) { + resetCtxOutputBuf(pRuntimeEnv); + } + + setCtxTagColumnInfo(pQuery, pRuntimeEnv->pCtx); + + // for loading block data in memory +// assert(vnodeList[pMeterObj->vnode].cfg.rowsInFileBlock == pMeterObj->pointsPerFileBlock); + return TSDB_CODE_SUCCESS; + + _error_clean: + tfree(pRuntimeEnv->resultInfo); + tfree(pRuntimeEnv->pCtx); + + return TSDB_CODE_SERV_OUT_OF_MEMORY; +} + +static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { + if (pRuntimeEnv->pQuery == NULL) { + return; + } + + SQuery *pQuery = pRuntimeEnv->pQuery; + + dTrace("QInfo:%p teardown runtime env", GET_QINFO_ADDR(pQuery)); + cleanupTimeWindowInfo(&pRuntimeEnv->windowResInfo, pQuery->numOfOutputCols); + + if (pRuntimeEnv->pCtx != NULL) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; + + for (int32_t j = 0; j < pCtx->numOfParams; ++j) { + tVariantDestroy(&pCtx->param[j]); + } + + tVariantDestroy(&pCtx->tag); + tfree(pCtx->tagInfo.pTagCtxList); + tfree(pRuntimeEnv->resultInfo[i].interResultBuf); + } + + tfree(pRuntimeEnv->resultInfo); + tfree(pRuntimeEnv->pCtx); + } + + taosDestoryInterpoInfo(&pRuntimeEnv->interpoInfo); + + if (pRuntimeEnv->pInterpoBuf != NULL) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + tfree(pRuntimeEnv->pInterpoBuf[i]); + } + + tfree(pRuntimeEnv->pInterpoBuf); + } + + destroyResultBuf(pRuntimeEnv->pResultBuf); + pRuntimeEnv->pTSBuf = tsBufDestory(pRuntimeEnv->pTSBuf); +} + +bool isQueryKilled(SQuery *pQuery) { + return false; + + SQInfo *pQInfo = (SQInfo *)GET_QINFO_ADDR(pQuery); +#if 0 + /* + * check if the queried meter is going to be deleted. + * if it will be deleted soon, stop current query ASAP. + */ + SMeterObj *pMeterObj = pQInfo->pObj; + if (vnodeIsMeterState(pMeterObj, TSDB_METER_STATE_DROPPING)) { + pQInfo->killed = 1; + return true; + } + + return (pQInfo->killed == 1); +#endif + return 0; +} + +bool isFixedOutputQuery(SQuery *pQuery) { + if (pQuery->intervalTime != 0) { + return false; + } + + // Note:top/bottom query is fixed output query + if (isTopBottomQuery(pQuery) || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + return true; + } + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SSqlFuncExprMsg *pExprMsg = &pQuery->pSelectExpr[i].pBase; + + // ignore the ts_comp function + if (i == 0 && pExprMsg->functionId == TSDB_FUNC_PRJ && pExprMsg->numOfParams == 1 && + pExprMsg->colInfo.colIdx == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + continue; + } + + if (pExprMsg->functionId == TSDB_FUNC_TS || pExprMsg->functionId == TSDB_FUNC_TS_DUMMY) { + continue; + } + + if (!IS_MULTIOUTPUT(aAggs[pExprMsg->functionId].nStatus)) { + return true; + } + } + + return false; +} + +bool isPointInterpoQuery(SQuery *pQuery) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionID = pQuery->pSelectExpr[i].pBase.functionId; + if (functionID == TSDB_FUNC_INTERP || functionID == TSDB_FUNC_LAST_ROW) { + return true; + } + } + + return false; +} + +// TODO REFACTOR:MERGE WITH CLIENT-SIDE FUNCTION +bool isSumAvgRateQuery(SQuery *pQuery) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + if (functionId == TSDB_FUNC_TS) { + continue; + } + + if (functionId == TSDB_FUNC_SUM_RATE || functionId == TSDB_FUNC_SUM_IRATE || functionId == TSDB_FUNC_AVG_RATE || + functionId == TSDB_FUNC_AVG_IRATE) { + return true; + } + } + + return false; +} + +bool isFirstLastRowQuery(SQuery *pQuery) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionID = pQuery->pSelectExpr[i].pBase.functionId; + if (functionID == TSDB_FUNC_LAST_ROW) { + return true; + } + } + + return false; +} + +bool notHasQueryTimeRange(SQuery *pQuery) { + return (pQuery->window.skey == 0 && pQuery->window.ekey == INT64_MAX && QUERY_IS_ASC_QUERY(pQuery)) || + (pQuery->window.skey == INT64_MAX && pQuery->window.ekey == 0 && (!QUERY_IS_ASC_QUERY(pQuery))); +} + +bool needSupplementaryScan(SQuery *pQuery) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG) { + continue; + } + + if (((functionId == TSDB_FUNC_LAST || functionId == TSDB_FUNC_LAST_DST) && QUERY_IS_ASC_QUERY(pQuery)) || + ((functionId == TSDB_FUNC_FIRST || functionId == TSDB_FUNC_FIRST_DST) && !QUERY_IS_ASC_QUERY(pQuery))) { + return true; + } + } + + return false; +} +///////////////////////////////////////////////////////////////////////////////////////////// + +void doGetAlignedIntervalQueryRangeImpl(SQuery *pQuery, int64_t key, int64_t keyFirst, int64_t keyLast, + int64_t *realSkey, int64_t *realEkey, STimeWindow* win) { + assert(key >= keyFirst && key <= keyLast && pQuery->slidingTime <= pQuery->intervalTime); + + win->skey = taosGetIntervalStartTimestamp(key, pQuery->slidingTime, pQuery->slidingTimeUnit, pQuery->precision); + + if (keyFirst > (INT64_MAX - pQuery->intervalTime)) { + /* + * if the realSkey > INT64_MAX - pQuery->intervalTime, the query duration between + * realSkey and realEkey must be less than one interval.Therefore, no need to adjust the query ranges. + */ + assert(keyLast - keyFirst < pQuery->intervalTime); + + *realSkey = keyFirst; + *realEkey = keyLast; + + win->ekey = INT64_MAX; + return; + } + + win->ekey = win->skey + pQuery->intervalTime - 1; + + if (win->skey < keyFirst) { + *realSkey = keyFirst; + } else { + *realSkey = win->skey; + } + + if (win->ekey < keyLast) { + *realEkey = win->ekey; + } else { + *realEkey = keyLast; + } +} + +static bool doGetQueryPos(TSKEY key, SQInfo *pQInfo, SPointInterpoSupporter *pPointInterpSupporter) { +#if 0 + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + SMeterObj * pMeterObj = pRuntimeEnv->pTabObj; + + /* key in query range. If not, no qualified in disk file */ + if (key != -1 && key <= pQuery->window.ekey) { + if (isPointInterpoQuery(pQuery)) { /* no qualified data in this query range */ + return getNeighborPoints(pQInfo, pMeterObj, pPointInterpSupporter); + } else { + return true; + } + } else { // key > pQuery->window.ekey, abort for normal query, continue for interp query + if (isPointInterpoQuery(pQuery)) { + return getNeighborPoints(pQInfo, pMeterObj, pPointInterpSupporter); + } else { + return false; + } + } +#endif +} + +static bool doSetDataInfo(SQInfo *pQInfo, SPointInterpoSupporter *pPointInterpSupporter, + void *pMeterObj, TSKEY nextKey) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + + if (isFirstLastRowQuery(pQuery)) { + /* + * if the pQuery->window.skey != pQuery->window.ekey for last_row query, + * the query range is existed, so set them both the value of nextKey + */ + if (pQuery->window.skey != pQuery->window.ekey) { + assert(pQuery->window.skey >= pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery) && nextKey >= pQuery->window.ekey && + nextKey <= pQuery->window.skey); + + pQuery->window.skey = nextKey; + pQuery->window.ekey = nextKey; + } + + return getNeighborPoints(pQInfo, pMeterObj, pPointInterpSupporter); + } else { + return true; + } +} + +// TODO refactor code, the best way to implement the last_row is utilizing the iterator +bool normalizeUnBoundLastRowQuery(SQInfo *pQInfo, SPointInterpoSupporter *pPointInterpSupporter) { +#if 0 + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + + SQuery * pQuery = pRuntimeEnv->pQuery; + SMeterObj *pMeterObj = pRuntimeEnv->pTabObj; + + assert(!QUERY_IS_ASC_QUERY(pQuery) && notHasQueryTimeRange(pQuery)); + __block_search_fn_t searchFn = vnodeSearchKeyFunc[pMeterObj->searchAlgorithm]; + + TSKEY lastKey = -1; + + pQuery->fileId = -1; + vnodeFreeFieldsEx(pRuntimeEnv); + + // keep in-memory cache status in local variables in case that it may be changed by write operation + getBasicCacheInfoSnapshot(pQuery, pMeterObj->pCache, pMeterObj->vnode); + + SCacheInfo *pCacheInfo = (SCacheInfo *)pMeterObj->pCache; + if (pCacheInfo != NULL && pCacheInfo->cacheBlocks != NULL && pQuery->numOfBlocks > 0) { + pQuery->fileId = -1; + TSKEY key = pMeterObj->lastKey; + + pQuery->window.skey = key; + pQuery->window.ekey = key; + pQuery->lastKey = pQuery->window.skey; + + /* + * cache block may have been flushed to disk, and no data in cache anymore. + * So, copy cache block to local buffer is required. + */ + lastKey = getQueryStartPositionInCache(pRuntimeEnv, &pQuery->slot, &pQuery->pos, false); + if (lastKey < 0) { // data has been flushed to disk, try again search in file + lastKey = getQueryPositionForCacheInvalid(pRuntimeEnv, searchFn); + + if (Q_STATUS_EQUAL(pQuery->status, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED)) { + return false; + } + } + } else { // no data in cache, try file + TSKEY key = pMeterObj->lastKeyOnFile; + + pQuery->window.skey = key; + pQuery->window.ekey = key; + pQuery->lastKey = pQuery->window.skey; + + bool ret = getQualifiedDataBlock(pMeterObj, pRuntimeEnv, QUERY_RANGE_LESS_EQUAL, searchFn); + if (!ret) { // no data in file, return false; + return false; + } + + lastKey = getTimestampInDiskBlock(pRuntimeEnv, pQuery->pos); + } + + assert(lastKey <= pQuery->window.skey); + + pQuery->window.skey = lastKey; + pQuery->window.ekey = lastKey; + pQuery->lastKey = pQuery->window.skey; + + return getNeighborPoints(pQInfo, pMeterObj, pPointInterpSupporter); +#endif + + return true; +} + +static void setScanLimitationByResultBuffer(SQuery *pQuery) { + if (isTopBottomQuery(pQuery)) { + pQuery->checkBufferInLoop = 0; + } else if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + pQuery->checkBufferInLoop = 0; + } else { + bool hasMultioutput = false; + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SSqlFuncExprMsg *pExprMsg = &pQuery->pSelectExpr[i].pBase; + if (pExprMsg->functionId == TSDB_FUNC_TS || pExprMsg->functionId == TSDB_FUNC_TS_DUMMY) { + continue; + } + + hasMultioutput = IS_MULTIOUTPUT(aAggs[pExprMsg->functionId].nStatus); + if (!hasMultioutput) { + break; + } + } + + pQuery->checkBufferInLoop = hasMultioutput ? 1 : 0; + } + + assert(0); +// pQuery->pointsOffset = pQuery->pointsToRead; +} + +/* + * todo add more parameters to check soon.. + */ +bool vnodeParametersSafetyCheck(SQuery *pQuery) { + // load data column information is incorrect + for (int32_t i = 0; i < pQuery->numOfCols - 1; ++i) { + if (pQuery->colList[i].info.colId == pQuery->colList[i + 1].info.colId) { + dError("QInfo:%p invalid data load column for query", GET_QINFO_ADDR(pQuery)); + return false; + } + } + return true; +} + +// todo ignore the avg/sum/min/max/count/stddev/top/bottom functions, of which +// the scan order is not matter +static bool onlyOneQueryType(SQuery *pQuery, int32_t functId, int32_t functIdDst) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + + if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG || + functionId == TSDB_FUNC_TAG_DUMMY) { + continue; + } + + if (functionId != functId && functionId != functIdDst) { + return false; + } + } + + return true; +} + +static bool onlyFirstQuery(SQuery *pQuery) { return onlyOneQueryType(pQuery, TSDB_FUNC_FIRST, TSDB_FUNC_FIRST_DST); } + +static bool onlyLastQuery(SQuery *pQuery) { return onlyOneQueryType(pQuery, TSDB_FUNC_LAST, TSDB_FUNC_LAST_DST); } + +static void changeExecuteScanOrder(SQuery *pQuery, bool metricQuery) { + // in case of point-interpolation query, use asc order scan + char msg[] = "QInfo:%p scan order changed for %s query, old:%d, new:%d, qrange exchanged, old qrange:%" PRId64 + "-%" PRId64 ", new qrange:%" PRId64 "-%" PRId64; + + // todo handle the case the the order irrelevant query type mixed up with order critical query type + // descending order query for last_row query + if (isFirstLastRowQuery(pQuery)) { + dTrace("QInfo:%p scan order changed for last_row query, old:%d, new:%d", GET_QINFO_ADDR(pQuery), + pQuery->order.order, TSQL_SO_DESC); + + pQuery->order.order = TSQL_SO_DESC; + + int64_t skey = MIN(pQuery->window.skey, pQuery->window.ekey); + int64_t ekey = MAX(pQuery->window.skey, pQuery->window.ekey); + + pQuery->window.skey = ekey; + pQuery->window.ekey = skey; + + return; + } + + if (isPointInterpoQuery(pQuery) && pQuery->intervalTime == 0) { + if (!QUERY_IS_ASC_QUERY(pQuery)) { + dTrace(msg, GET_QINFO_ADDR(pQuery), "interp", pQuery->order.order, TSQL_SO_ASC, pQuery->window.skey, pQuery->window.ekey, + pQuery->window.ekey, pQuery->window.skey); + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + } + + pQuery->order.order = TSQL_SO_ASC; + return; + } + + if (pQuery->intervalTime == 0) { + if (onlyFirstQuery(pQuery)) { + if (!QUERY_IS_ASC_QUERY(pQuery)) { + dTrace(msg, GET_QINFO_ADDR(pQuery), "only-first", pQuery->order.order, TSQL_SO_ASC, pQuery->window.skey, pQuery->window.ekey, + pQuery->window.ekey, pQuery->window.skey); + + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + } + + pQuery->order.order = TSQL_SO_ASC; + } else if (onlyLastQuery(pQuery)) { + if (QUERY_IS_ASC_QUERY(pQuery)) { + dTrace(msg, GET_QINFO_ADDR(pQuery), "only-last", pQuery->order.order, TSQL_SO_DESC, pQuery->window.skey, pQuery->window.ekey, + pQuery->window.ekey, pQuery->window.skey); + + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + } + + pQuery->order.order = TSQL_SO_DESC; + } + + } else { // interval query + if (metricQuery) { + if (onlyFirstQuery(pQuery)) { + if (!QUERY_IS_ASC_QUERY(pQuery)) { + dTrace(msg, GET_QINFO_ADDR(pQuery), "only-first stable", pQuery->order.order, TSQL_SO_ASC, pQuery->window.skey, + pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); + + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + } + + pQuery->order.order = TSQL_SO_ASC; + } else if (onlyLastQuery(pQuery)) { + if (QUERY_IS_ASC_QUERY(pQuery)) { + dTrace(msg, GET_QINFO_ADDR(pQuery), "only-last stable", pQuery->order.order, TSQL_SO_DESC, pQuery->window.skey, + pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); + + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + } + + pQuery->order.order = TSQL_SO_DESC; + } + } + } +} + +static void doSetInterpVal(SQLFunctionCtx *pCtx, TSKEY ts, int16_t type, int32_t index, char *data) { + assert(pCtx->param[index].pz == NULL); + + int32_t len = 0; + size_t t = 0; + + if (type == TSDB_DATA_TYPE_BINARY) { + t = strlen(data); + + len = t + 1 + TSDB_KEYSIZE; + pCtx->param[index].pz = calloc(1, len); + } else if (type == TSDB_DATA_TYPE_NCHAR) { + t = wcslen((const wchar_t *)data); + + len = (t + 1) * TSDB_NCHAR_SIZE + TSDB_KEYSIZE; + pCtx->param[index].pz = calloc(1, len); + } else { + len = TSDB_KEYSIZE * 2; + pCtx->param[index].pz = malloc(len); + } + + pCtx->param[index].nType = TSDB_DATA_TYPE_BINARY; + + char *z = pCtx->param[index].pz; + *(TSKEY *)z = ts; + z += TSDB_KEYSIZE; + + switch (type) { + case TSDB_DATA_TYPE_FLOAT: + *(double *)z = GET_FLOAT_VAL(data); + break; + case TSDB_DATA_TYPE_DOUBLE: + *(double *)z = GET_DOUBLE_VAL(data); + break; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_TIMESTAMP: + *(int64_t *)z = GET_INT64_VAL(data); + break; + case TSDB_DATA_TYPE_BINARY: + strncpy(z, data, t); + break; + case TSDB_DATA_TYPE_NCHAR: { + wcsncpy((wchar_t *)z, (const wchar_t *)data, t); + } break; + default: + assert(0); + } + + pCtx->param[index].nLen = len; +} + +/** + * param[1]: default value/previous value of specified timestamp + * param[2]: next value of specified timestamp + * param[3]: denotes if the result is a precious result or interpolation results + * + * @param pQInfo + * @param pQInfo + * @param pInterpoRaw + */ +void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointInterpSupport) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery *pQuery = pRuntimeEnv->pQuery; + + // not point interpolation query, abort + if (!isPointInterpoQuery(pQuery)) { + return; + } + + int32_t count = 1; + TSKEY key = *(TSKEY *)pPointInterpSupport->pNextPoint[0]; + + if (key == pQuery->window.skey) { + // the queried timestamp has value, return it directly without interpolation + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + tVariantCreateFromBinary(&pRuntimeEnv->pCtx[i].param[3], (char *)&count, sizeof(count), TSDB_DATA_TYPE_INT); + + pRuntimeEnv->pCtx[i].param[0].i64Key = key; + pRuntimeEnv->pCtx[i].param[0].nType = TSDB_DATA_TYPE_BIGINT; + } + } else { + // set the direct previous(next) point for process + count = 2; + + if (pQuery->interpoType == TSDB_INTERPO_SET_VALUE) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; + + // only the function of interp needs the corresponding information + if (pCtx->functionId != TSDB_FUNC_INTERP) { + continue; + } + + pCtx->numOfParams = 4; + + SInterpInfo *pInterpInfo = (SInterpInfo *)pRuntimeEnv->pCtx[i].aOutputBuf; + pInterpInfo->pInterpDetail = calloc(1, sizeof(SInterpInfoDetail)); + + SInterpInfoDetail *pInterpDetail = pInterpInfo->pInterpDetail; + + // for primary timestamp column, set the flag + if (pQuery->pSelectExpr[i].pBase.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + pInterpDetail->primaryCol = 1; + } + + tVariantCreateFromBinary(&pCtx->param[3], (char *)&count, sizeof(count), TSDB_DATA_TYPE_INT); + + if (isNull((char *)&pQuery->defaultVal[i], pCtx->inputType)) { + pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; + } else { + tVariantCreateFromBinary(&pCtx->param[1], (char *)&pQuery->defaultVal[i], pCtx->inputBytes, pCtx->inputType); + } + + pInterpDetail->ts = pQuery->window.skey; + pInterpDetail->type = pQuery->interpoType; + } + } else { + TSKEY prevKey = *(TSKEY *)pPointInterpSupport->pPrevPoint[0]; + TSKEY nextKey = *(TSKEY *)pPointInterpSupport->pNextPoint[0]; + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; + + // tag column does not need the interp environment + if (pQuery->pSelectExpr[i].pBase.functionId == TSDB_FUNC_TAG) { + continue; + } + + int32_t colInBuf = pQuery->pSelectExpr[i].pBase.colInfo.colIdxInBuf; + + SInterpInfo *pInterpInfo = (SInterpInfo *)pRuntimeEnv->pCtx[i].aOutputBuf; + + pInterpInfo->pInterpDetail = calloc(1, sizeof(SInterpInfoDetail)); + SInterpInfoDetail *pInterpDetail = pInterpInfo->pInterpDetail; + +// int32_t type = GET_COLUMN_TYPE(pQuery, i); + int32_t type = 0; + assert(0); + + // for primary timestamp column, set the flag + if (pQuery->pSelectExpr[i].pBase.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + pInterpDetail->primaryCol = 1; + } else { + doSetInterpVal(pCtx, prevKey, type, 1, pPointInterpSupport->pPrevPoint[colInBuf]); + doSetInterpVal(pCtx, nextKey, type, 2, pPointInterpSupport->pNextPoint[colInBuf]); + } + + tVariantCreateFromBinary(&pRuntimeEnv->pCtx[i].param[3], (char *)&count, sizeof(count), TSDB_DATA_TYPE_INT); + + pInterpDetail->ts = pQInfo->runtimeEnv.pQuery->window.skey; + pInterpDetail->type = pQuery->interpoType; + } + } + } +} + +void pointInterpSupporterInit(SQuery *pQuery, SPointInterpoSupporter *pInterpoSupport) { + if (isPointInterpoQuery(pQuery)) { + pInterpoSupport->pPrevPoint = malloc(pQuery->numOfCols * POINTER_BYTES); + pInterpoSupport->pNextPoint = malloc(pQuery->numOfCols * POINTER_BYTES); + + pInterpoSupport->numOfCols = pQuery->numOfCols; + + /* get appropriated size for one row data source*/ + int32_t len = 0; + for (int32_t i = 0; i < pQuery->numOfCols; ++i) { + len += pQuery->colList[i].info.bytes; + } + +// assert(PRIMARY_TSCOL_LOADED(pQuery)); + + void *prev = calloc(1, len); + void *next = calloc(1, len); + + int32_t offset = 0; + + for (int32_t i = 0; i < pQuery->numOfCols; ++i) { + pInterpoSupport->pPrevPoint[i] = prev + offset; + pInterpoSupport->pNextPoint[i] = next + offset; + + offset += pQuery->colList[i].info.bytes; + } + } +} + +void pointInterpSupporterDestroy(SPointInterpoSupporter *pPointInterpSupport) { + if (pPointInterpSupport->numOfCols <= 0 || pPointInterpSupport->pPrevPoint == NULL) { + return; + } + + tfree(pPointInterpSupport->pPrevPoint[0]); + tfree(pPointInterpSupport->pNextPoint[0]); + + tfree(pPointInterpSupport->pPrevPoint); + tfree(pPointInterpSupport->pNextPoint); + + pPointInterpSupport->numOfCols = 0; +} + +static void allocMemForInterpo(SQInfo *pQInfo, SQuery *pQuery, void *pMeterObj) { +#if 0 + if (pQuery->interpoType != TSDB_INTERPO_NONE) { + assert(isIntervalQuery(pQuery) || (pQuery->intervalTime == 0 && isPointInterpoQuery(pQuery))); + + if (isIntervalQuery(pQuery)) { + pQInfo->runtimeEnv.pInterpoBuf = malloc(POINTER_BYTES * pQuery->numOfOutputCols); + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + pQInfo->runtimeEnv.pInterpoBuf[i] = + calloc(1, sizeof(tFilePage) + pQuery->pSelectExpr[i].resBytes * pMeterObj->pointsPerFileBlock); + } + } + } +#endif +} + +static int32_t getInitialPageNum(SQInfo *pQInfo) { + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + int32_t INITIAL_RESULT_ROWS_VALUE = 16; + + int32_t num = 0; + + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + num = 128; + } else if (isIntervalQuery(pQuery)) { // time window query, allocate one page for each table + size_t s = taosHashGetSize(pQInfo->pTableList); + num = MAX(s, INITIAL_RESULT_ROWS_VALUE); + } else { // for super table query, one page for each subset + num = pQInfo->pSidSet->numOfSubSet; + } + + assert(num > 0); + return num; +} + +static int32_t getRowParamForMultiRowsOutput(SQuery *pQuery, bool isSTableQuery) { + int32_t rowparam = 1; + + if (isTopBottomQuery(pQuery) && (!isSTableQuery)) { + rowparam = pQuery->pSelectExpr[1].pBase.arg->argValue.i64; + } + + return rowparam; +} + +static int32_t getNumOfRowsInResultPage(SQuery *pQuery, bool isSTableQuery) { + int32_t rowSize = pQuery->rowSize * getRowParamForMultiRowsOutput(pQuery, isSTableQuery); + return (DEFAULT_INTERN_BUF_SIZE - sizeof(tFilePage)) / rowSize; +} + +char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult) { + assert(pResult != NULL && pRuntimeEnv != NULL); + + SQuery * pQuery = pRuntimeEnv->pQuery; + tFilePage *page = getResultBufferPageById(pRuntimeEnv->pResultBuf, pResult->pos.pageId); + + int32_t numOfRows = getNumOfRowsInResultPage(pQuery, pRuntimeEnv->stableQuery); + int32_t realRowId = pResult->pos.rowId * getRowParamForMultiRowsOutput(pQuery, pRuntimeEnv->stableQuery); + + return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * numOfRows + + pQuery->pSelectExpr[columnIndex].resBytes * realRowId; +} + +void vnodeQueryFreeQInfoEx(SQInfo *pQInfo) { + if (pQInfo == NULL) { + return; + } + + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); + + if (pQInfo->pTableList != NULL) { + taosHashCleanup(pQInfo->pTableList); + pQInfo->pTableList = NULL; + } + +// tSidSetDestroy(&pQInfo->pSidSet); + + if (pQInfo->pTableDataInfo != NULL) { + size_t num = taosHashGetSize(pQInfo->pTableList); + for (int32_t j = 0; j < num; ++j) { + destroyMeterQueryInfo(pQInfo->pTableDataInfo[j].pTableQInfo, pQuery->numOfOutputCols); + } + } + + tfree(pQInfo->pTableDataInfo); +} + +int32_t vnodeSTableQueryPrepare(SQInfo *pQInfo, SQuery *pQuery, void *param) { + if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.skey > pQuery->window.ekey)) || + (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.ekey > pQuery->window.skey))) { + dTrace("QInfo:%p no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo, pQuery->window.skey, pQuery->window.ekey, + pQuery->order.order); + + sem_post(&pQInfo->dataReady); +// pQInfo->over = 1; + + return TSDB_CODE_SUCCESS; + } + + pQuery->status = 0; + + pQInfo->rec = (SResultRec) {0}; + pQuery->rec = (SResultRec) {0}; + + changeExecuteScanOrder(pQuery, true); + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + + /* + * since we employ the output control mechanism in main loop. + * so, disable it during data block scan procedure. + */ + setScanLimitationByResultBuffer(pQuery); + + // save raw query range for applying to each subgroup + pQuery->lastKey = pQuery->window.skey; + + // create runtime environment + SColumnModel *pTagSchemaInfo = pQInfo->pSidSet->pColumnModel; + + // get one queried meter + assert(0); +// SMeterObj *pMeter = getMeterObj(pQInfo->pTableList, pQInfo->pSidSet->pSids[0]->sid); + + pRuntimeEnv->pTSBuf = param; + pRuntimeEnv->cur.vnodeIndex = -1; + + // set the ts-comp file traverse order + if (param != NULL) { + int16_t order = (pQuery->order.order == pRuntimeEnv->pTSBuf->tsOrder) ? TSQL_SO_ASC : TSQL_SO_DESC; + tsBufSetTraverseOrder(pRuntimeEnv->pTSBuf, order); + } + + assert(0); +// int32_t ret = setupQueryRuntimeEnv(pMeter, pQuery, &pQInfo->runtimeEnv, pTagSchemaInfo, TSQL_SO_ASC, true); +// if (ret != TSDB_CODE_SUCCESS) { +// return ret; +// } + +// tSidSetSort(pQInfo->pSidSet); + + int32_t size = getInitialPageNum(pQInfo); + int32_t ret = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, size, pQuery->rowSize); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + + if (pQuery->intervalTime == 0) { + int16_t type = TSDB_DATA_TYPE_NULL; + + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // group by columns not tags; + type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); + } else { + type = TSDB_DATA_TYPE_INT; // group id + } + + initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, 512, 4096, type); + } + + pRuntimeEnv->numOfRowsPerPage = getNumOfRowsInResultPage(pQuery, true); + + STsdbQueryCond cond = {0}; + cond.twindow = (STimeWindow){.skey = pQuery->window.skey, .ekey = pQuery->window.ekey}; + cond.order = pQuery->order.order; + + cond.colList = *pQuery->colList; + SArray *sa = taosArrayInit(1, POINTER_BYTES); + + for(int32_t i = 0; i < pQInfo->pSidSet->numOfSids; ++i) { +// SMeterObj *p1 = getMeterObj(pQInfo->pTableList, pQInfo->pSidSet->pSids[i]->sid); +// taosArrayPush(sa, &p1); + } + + SArray *cols = taosArrayInit(pQuery->numOfCols, sizeof(pQuery->colList[0])); + for (int32_t i = 0; i < pQuery->numOfCols; ++i) { + taosArrayPush(cols, &pQuery->colList[i]); + } + + pRuntimeEnv->pQueryHandle = tsdbQueryByTableId(&cond, sa, cols); + + // metric query do not invoke interpolation, it will be done at the second-stage merge + if (!isPointInterpoQuery(pQuery)) { + pQuery->interpoType = TSDB_INTERPO_NONE; + } + + TSKEY revisedStime = taosGetIntervalStartTimestamp(pQuery->window.skey, pQuery->intervalTime, + pQuery->slidingTimeUnit, pQuery->precision); + taosInitInterpoInfo(&pRuntimeEnv->interpoInfo, pQuery->order.order, revisedStime, 0, 0); + pRuntimeEnv->stableQuery = true; + + return TSDB_CODE_SUCCESS; +} + +/** + * decrease the refcount for each table involved in this query + * @param pQInfo + */ +void vnodeDecMeterRefcnt(SQInfo *pQInfo) { + if (pQInfo != NULL) { + assert(taosHashGetSize(pQInfo->pTableList) >= 1); + } + +#if 0 + if (pQInfo == NULL || pQInfo->numOfMeters == 1) { + atomic_fetch_sub_32(&pQInfo->pObj->numOfQueries, 1); + dTrace("QInfo:%p vid:%d sid:%d meterId:%s, query is over, numOfQueries:%d", pQInfo, pQInfo->pObj->vnode, + pQInfo->pObj->sid, pQInfo->pObj->meterId, pQInfo->pObj->numOfQueries); + } else { + int32_t num = 0; + for (int32_t i = 0; i < pQInfo->numOfMeters; ++i) { + SMeterObj *pMeter = getMeterObj(pQInfo->pTableList, pQInfo->pSidSet->pSids[i]->sid); + atomic_fetch_sub_32(&(pMeter->numOfQueries), 1); + + if (pMeter->numOfQueries > 0) { + dTrace("QInfo:%p vid:%d sid:%d meterId:%s, query is over, numOfQueries:%d", pQInfo, pMeter->vnode, pMeter->sid, + pMeter->meterId, pMeter->numOfQueries); + num++; + } + } + + /* + * in order to reduce log output, for all meters of which numOfQueries count are 0, + * we do not output corresponding information + */ + num = pQInfo->numOfMeters - num; + dTrace("QInfo:%p metric query is over, dec query ref for %d meters, numOfQueries on %d meters are 0", pQInfo, + pQInfo->numOfMeters, num); + } +#endif +} + +void setTimestampRange(SQueryRuntimeEnv *pRuntimeEnv, int64_t stime, int64_t etime) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + + if (functionId == TSDB_FUNC_SPREAD) { + pRuntimeEnv->pCtx[i].param[1].dKey = stime; + pRuntimeEnv->pCtx[i].param[2].dKey = etime; + + pRuntimeEnv->pCtx[i].param[1].nType = TSDB_DATA_TYPE_DOUBLE; + pRuntimeEnv->pCtx[i].param[2].nType = TSDB_DATA_TYPE_DOUBLE; + } + } +} + +static bool needToLoadDataBlock(SQuery *pQuery, SDataStatis *pDataStatis, SQLFunctionCtx *pCtx, + int32_t numOfTotalPoints) { + if (pDataStatis == NULL) { + return true; + } + +#if 0 + for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { + SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; + int32_t colIndex = pFilterInfo->info.colIdx; + + // this column not valid in current data block + if (colIndex < 0 || pDataStatis[colIndex].colId != pFilterInfo->info.data.colId) { + continue; + } + + // not support pre-filter operation on binary/nchar data type + if (!vnodeSupportPrefilter(pFilterInfo->info.data.type)) { + continue; + } + + // all points in current column are NULL, no need to check its boundary value + if (pDataStatis[colIndex].numOfNull == numOfTotalPoints) { + continue; + } + + if (pFilterInfo->info.info.type == TSDB_DATA_TYPE_FLOAT) { + float minval = *(double *)(&pDataStatis[colIndex].min); + float maxval = *(double *)(&pDataStatis[colIndex].max); + + for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) { + if (pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&minval, (char *)&maxval)) { + return true; + } + } + } else { + for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) { + if (pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&pDataStatis[colIndex].min, + (char *)&pDataStatis[colIndex].max)) { + return true; + } + } + } + } + + // todo disable this opt code block temporarily + // for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + // int32_t functId = pQuery->pSelectExpr[i].pBase.functionId; + // if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { + // return top_bot_datablock_filter(&pCtx[i], functId, (char *)&pField[i].min, (char *)&pField[i].max); + // } + // } + +#endif + return true; +} + +// previous time window may not be of the same size of pQuery->intervalTime +static void getNextTimeWindow(SQuery *pQuery, STimeWindow *pTimeWindow) { + int32_t factor = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + + pTimeWindow->skey += (pQuery->slidingTime * factor); + pTimeWindow->ekey = pTimeWindow->skey + (pQuery->intervalTime - 1); +} + +SArray* loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pBlockInfo, SDataStatis** pStatis) { + SQuery* pQuery = pRuntimeEnv->pQuery; + tsdb_query_handle_t pQueryHandle = pRuntimeEnv->pQueryHandle; + + uint32_t r = 0; + SArray * pDataBlock = NULL; + +// STimeWindow *w = &pQueryHandle->window; + + if (pQuery->numOfFilterCols > 0) { + r = BLK_DATA_ALL_NEEDED; + } else { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + int32_t colId = pQuery->pSelectExpr[i].pBase.colInfo.colId; + +// r |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], w->skey, w->ekey, colId); + } + + if (pRuntimeEnv->pTSBuf > 0 || isIntervalQuery(pQuery)) { + r |= BLK_DATA_ALL_NEEDED; + } + } + + if (r == BLK_DATA_NO_NEEDED) { + // qTrace("QInfo:%p vid:%d sid:%d id:%s, slot:%d, data block ignored, brange:%" PRId64 "-%" PRId64 ", + // rows:%d", GET_QINFO_ADDR(pQuery), pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->slot, + // pBlock->keyFirst, pBlock->keyLast, pBlock->numOfPoints); + } else if (r == BLK_DATA_FILEDS_NEEDED) { + if (tsdbRetrieveDataBlockStatisInfo(pRuntimeEnv->pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { + // return DISK_DATA_LOAD_FAILED; + } + + if (pStatis == NULL) { + pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); + } + } else { + assert(r == BLK_DATA_ALL_NEEDED); + if (tsdbRetrieveDataBlockStatisInfo(pRuntimeEnv->pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { + // return DISK_DATA_LOAD_FAILED; + } + + /* + * if this block is completed included in the query range, do more filter operation + * filter the data block according to the value filter condition. + * no need to load the data block, continue for next block + */ + if (!needToLoadDataBlock(pQuery, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->size)) { +#if defined(_DEBUG_VIEW) + dTrace("QInfo:%p fileId:%d, slot:%d, block discarded by per-filter", GET_QINFO_ADDR(pQuery), pQuery->fileId, + pQuery->slot); +#endif + // return DISK_DATA_DISCARDED; + } + + pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); + } + + return pDataBlock; +} + +static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { +#if 0 + SQuery *pQuery = pRuntimeEnv->pQuery; + assert(0); +// __block_search_fn_t searchFn = vnodeSearchKeyFunc[pRuntimeEnv->pTabObj->searchAlgorithm]; + + int64_t cnt = 0; + dTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", lastkey:%" PRId64 ", order:%d", GET_QINFO_ADDR(pQuery), + pQuery->window.skey, pQuery->window.ekey, pQuery->lastKey, pQuery->order.order); + + tsdb_query_handle_t pQueryHandle = pRuntimeEnv->pQueryHandle; + + while (tsdbNextDataBlock(pQueryHandle)) { + // check if query is killed or not set the status of query to pass the status check + if (isQueryKilled(pQuery)) { + setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); + return cnt; + } + + SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle); + + if (isIntervalQuery(pQuery) && pRuntimeEnv->windowResInfo.prevSKey == 0) { + TSKEY skey1, ekey1; + STimeWindow w = {0}; + SWindowResInfo* pWindowResInfo = &pRuntimeEnv->windowResInfo; + + if (QUERY_IS_ASC_QUERY(pQuery)) { +// doGetAlignedIntervalQueryRangeImpl(pQuery, blockInfo.window.skey, blockInfo.window.skey, +// pQueryHandle->window.ekey, &skey1, &ekey1, &w); + pWindowResInfo->startTime = w.skey; + pWindowResInfo->prevSKey = w.skey; + } else { + // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp + TSKEY winStart = blockInfo.window.ekey - pQuery->intervalTime; +// doGetAlignedIntervalQueryRangeImpl(pQuery, winStart, pQueryHandle->window.ekey, +// blockInfo.window.ekey, &skey1, &ekey1, &w); + +// pWindowResInfo->startTime = pQueryHandle->window.skey; + pWindowResInfo->prevSKey = w.skey; + } + } + + int32_t numOfRes = 0; + + SDataStatis *pStatis = NULL; + SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); +// int32_t forwardStep = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, searchFn, &numOfRes, +// &pRuntimeEnv->windowResInfo, pDataBlock); + +// dTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", fileId:%d, slot:%d, pos:%d, rows:%d, checked:%d", +// GET_QINFO_ADDR(pQuery), blockInfo.window.skey, blockInfo.window.ekey, pQueryHandle->cur.fileId, pQueryHandle->cur.slot, +// pQuery->pos, blockInfo.size, forwardStep); + + // save last access position +// cnt += forwardStep; + +// if (queryPaused(pQuery, &blockInfo, forwardStep)) { +// if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { +// break; +// } + } + } + + // if the result buffer is not full, set the query completed flag + if (!Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { + setQueryStatus(pQuery, QUERY_COMPLETED); + } + + if (isIntervalQuery(pQuery) && IS_MASTER_SCAN(pRuntimeEnv)) { + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)) { + int32_t step = QUERY_IS_ASC_QUERY(pQuery)? QUERY_ASC_FORWARD_STEP:QUERY_DESC_FORWARD_STEP; + + closeAllTimeWindow(&pRuntimeEnv->windowResInfo); + removeRedundantWindow(&pRuntimeEnv->windowResInfo, pQuery->lastKey - step, step); + pRuntimeEnv->windowResInfo.curIndex = pRuntimeEnv->windowResInfo.size - 1; + } else { + assert(Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)); + } + } + + return cnt; +#endif + return 0; +} + +static void updatelastkey(SQuery *pQuery, STableQueryInfo *pTableQInfo) { pTableQInfo->lastKey = pQuery->lastKey; } + +/* + * set tag value in SQLFunctionCtx + * e.g.,tag information into input buffer + */ +static void doSetTagValueInParam(SColumnModel *pTagSchema, int32_t tagColIdx, void *pMeterSidInfo, + tVariant *param) { + assert(tagColIdx >= 0); +#if 0 + int16_t offset = getColumnModelOffset(pTagSchema, tagColIdx); + + void * pStr = (char *)pMeterSidInfo->tags + offset; + SSchema *pCol = getColumnModelSchema(pTagSchema, tagColIdx); + + tVariantDestroy(param); + + if (isNull(pStr, pCol->type)) { + param->nType = TSDB_DATA_TYPE_NULL; + } else { + tVariantCreateFromBinary(param, pStr, pCol->bytes, pCol->type); + } +#endif +} + +void vnodeSetTagValueInParam(tSidSet *pSidSet, SQueryRuntimeEnv *pRuntimeEnv, void *pMeterSidInfo) { + SQuery * pQuery = pRuntimeEnv->pQuery; + SColumnModel *pTagSchema = pSidSet->pColumnModel; + + SSqlFuncExprMsg *pFuncMsg = &pQuery->pSelectExpr[0].pBase; + if (pQuery->numOfOutputCols == 1 && pFuncMsg->functionId == TSDB_FUNC_TS_COMP) { + assert(pFuncMsg->numOfParams == 1); + doSetTagValueInParam(pTagSchema, pFuncMsg->arg->argValue.i64, pMeterSidInfo, &pRuntimeEnv->pCtx[0].tag); + } else { + // set tag value, by which the results are aggregated. + for (int32_t idx = 0; idx < pQuery->numOfOutputCols; ++idx) { + SColIndexEx *pColEx = &pQuery->pSelectExpr[idx].pBase.colInfo; + + // ts_comp column required the tag value for join filter + if (!TSDB_COL_IS_TAG(pColEx->flag)) { + continue; + } + + doSetTagValueInParam(pTagSchema, pColEx->colIdx, pMeterSidInfo, &pRuntimeEnv->pCtx[idx].tag); + } + + // set the join tag for first column + SSqlFuncExprMsg *pFuncMsg = &pQuery->pSelectExpr[0].pBase; + if (pFuncMsg->functionId == TSDB_FUNC_TS && pFuncMsg->colInfo.colIdx == PRIMARYKEY_TIMESTAMP_COL_INDEX && + pRuntimeEnv->pTSBuf != NULL) { + assert(pFuncMsg->numOfParams == 1); + doSetTagValueInParam(pTagSchema, pFuncMsg->arg->argValue.i64, pMeterSidInfo, &pRuntimeEnv->pCtx[0].tag); + } + } +} + +static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowResult *pWindowRes, bool mergeFlag) { + SQuery * pQuery = pRuntimeEnv->pQuery; + SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + if (!mergeFlag) { + pCtx[i].aOutputBuf = pCtx[i].aOutputBuf + pCtx[i].outputBytes; + pCtx[i].currentStage = FIRST_STAGE_MERGE; + + resetResultInfo(pCtx[i].resultInfo); + aAggs[functionId].init(&pCtx[i]); + } + + pCtx[i].hasNull = true; + pCtx[i].nStartQueryTimestamp = timestamp; + pCtx[i].aInputElemBuf = getPosInResultPage(pRuntimeEnv, i, pWindowRes); + // pCtx[i].aInputElemBuf = ((char *)inputSrc->data) + + // ((int32_t)pRuntimeEnv->offset[i] * pRuntimeEnv->numOfRowsPerPage) + + // pCtx[i].outputBytes * inputIdx; + + // in case of tag column, the tag information should be extracted from input buffer + if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TAG) { + tVariantDestroy(&pCtx[i].tag); + tVariantCreateFromBinary(&pCtx[i].tag, pCtx[i].aInputElemBuf, pCtx[i].inputBytes, pCtx[i].inputType); + } + } + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + if (functionId == TSDB_FUNC_TAG_DUMMY) { + continue; + } + + aAggs[functionId].distMergeFunc(&pCtx[i]); + } +} + +static void printBinaryData(int32_t functionId, char *data, int32_t srcDataType) { + if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_LAST_DST) { + switch (srcDataType) { + case TSDB_DATA_TYPE_BINARY: + printf("%" PRId64 ",%s\t", *(TSKEY *)data, (data + TSDB_KEYSIZE + 1)); + break; + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_BOOL: + printf("%" PRId64 ",%d\t", *(TSKEY *)data, *(int8_t *)(data + TSDB_KEYSIZE + 1)); + break; + case TSDB_DATA_TYPE_SMALLINT: + printf("%" PRId64 ",%d\t", *(TSKEY *)data, *(int16_t *)(data + TSDB_KEYSIZE + 1)); + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + printf("%" PRId64 ",%" PRId64 "\t", *(TSKEY *)data, *(TSKEY *)(data + TSDB_KEYSIZE + 1)); + break; + case TSDB_DATA_TYPE_INT: + printf("%" PRId64 ",%d\t", *(TSKEY *)data, *(int32_t *)(data + TSDB_KEYSIZE + 1)); + break; + case TSDB_DATA_TYPE_FLOAT: + printf("%" PRId64 ",%f\t", *(TSKEY *)data, *(float *)(data + TSDB_KEYSIZE + 1)); + break; + case TSDB_DATA_TYPE_DOUBLE: + printf("%" PRId64 ",%lf\t", *(TSKEY *)data, *(double *)(data + TSDB_KEYSIZE + 1)); + break; + } + } else if (functionId == TSDB_FUNC_AVG) { + printf("%lf,%d\t", *(double *)data, *(int32_t *)(data + sizeof(double))); + } else if (functionId == TSDB_FUNC_SPREAD) { + printf("%lf,%lf\t", *(double *)data, *(double *)(data + sizeof(double))); + } else if (functionId == TSDB_FUNC_TWA) { + data += 1; + printf("%lf,%" PRId64 ",%" PRId64 ",%" PRId64 "\t", *(double *)data, *(int64_t *)(data + 8), + *(int64_t *)(data + 16), *(int64_t *)(data + 24)); + } else if (functionId == TSDB_FUNC_MIN || functionId == TSDB_FUNC_MAX) { + switch (srcDataType) { + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_BOOL: + printf("%d\t", *(int8_t *)data); + break; + case TSDB_DATA_TYPE_SMALLINT: + printf("%d\t", *(int16_t *)data); + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + printf("%" PRId64 "\t", *(int64_t *)data); + break; + case TSDB_DATA_TYPE_INT: + printf("%d\t", *(int *)data); + break; + case TSDB_DATA_TYPE_FLOAT: + printf("%f\t", *(float *)data); + break; + case TSDB_DATA_TYPE_DOUBLE: + printf("%f\t", *(float *)data); + break; + } + } else if (functionId == TSDB_FUNC_SUM) { + if (srcDataType == TSDB_DATA_TYPE_FLOAT || srcDataType == TSDB_DATA_TYPE_DOUBLE) { + printf("%lf\t", *(float *)data); + } else { + printf("%" PRId64 "\t", *(int64_t *)data); + } + } else { + printf("%s\t", data); + } +} + +void UNUSED_FUNC displayInterResult(SData **pdata, SQuery *pQuery, int32_t numOfRows) { +#if 0 + int32_t numOfCols = pQuery->numOfOutputCols; + printf("super table query intermediate result, total:%d\n", numOfRows); + + SQInfo * pQInfo = (SQInfo *)(GET_QINFO_ADDR(pQuery)); + SMeterObj *pMeterObj = pQInfo->pObj; + + for (int32_t j = 0; j < numOfRows; ++j) { + for (int32_t i = 0; i < numOfCols; ++i) { + switch (pQuery->pSelectExpr[i].resType) { + case TSDB_DATA_TYPE_BINARY: { + int32_t colIdx = pQuery->pSelectExpr[i].pBase.colInfo.colIdx; + int32_t type = 0; + + if (TSDB_COL_IS_TAG(pQuery->pSelectExpr[i].pBase.colInfo.flag)) { + type = pQuery->pSelectExpr[i].resType; + } else { + type = pMeterObj->schema[colIdx].type; + } + printBinaryData(pQuery->pSelectExpr[i].pBase.functionId, pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j, + type); + break; + } + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_BIGINT: + printf("%" PRId64 "\t", *(int64_t *)(pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j)); + break; + case TSDB_DATA_TYPE_INT: + printf("%d\t", *(int32_t *)(pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j)); + break; + case TSDB_DATA_TYPE_FLOAT: + printf("%f\t", *(float *)(pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j)); + break; + case TSDB_DATA_TYPE_DOUBLE: + printf("%lf\t", *(double *)(pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j)); + break; + } + } + printf("\n"); + } +#endif +} + +typedef struct SCompSupporter { + STableDataInfo ** pTableDataInfo; + int32_t * position; + SQInfo *pQInfo; +} SCompSupporter; + +int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) { + int32_t left = *(int32_t *)pLeft; + int32_t right = *(int32_t *)pRight; + + SCompSupporter * supporter = (SCompSupporter *)param; + SQueryRuntimeEnv *pRuntimeEnv = &supporter->pQInfo->runtimeEnv; + + int32_t leftPos = supporter->position[left]; + int32_t rightPos = supporter->position[right]; + + /* left source is exhausted */ + if (leftPos == -1) { + return 1; + } + + /* right source is exhausted*/ + if (rightPos == -1) { + return -1; + } + + SWindowResInfo *pWindowResInfo1 = &supporter->pTableDataInfo[left]->pTableQInfo->windowResInfo; + SWindowResult * pWindowRes1 = getWindowResult(pWindowResInfo1, leftPos); + + char *b1 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes1); + TSKEY leftTimestamp = GET_INT64_VAL(b1); + + SWindowResInfo *pWindowResInfo2 = &supporter->pTableDataInfo[right]->pTableQInfo->windowResInfo; + SWindowResult * pWindowRes2 = getWindowResult(pWindowResInfo2, rightPos); + + char *b2 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes2); + TSKEY rightTimestamp = GET_INT64_VAL(b2); + + if (leftTimestamp == rightTimestamp) { + return 0; + } + + return leftTimestamp > rightTimestamp ? 1 : -1; +} + +int32_t mergeMetersResultToOneGroups(SQInfo *pQInfo) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + + int64_t st = taosGetTimestampMs(); + int32_t ret = TSDB_CODE_SUCCESS; + + while (pQInfo->subgroupIdx < pQInfo->pSidSet->numOfSubSet) { + int32_t start = pQInfo->pSidSet->starterPos[pQInfo->subgroupIdx]; + int32_t end = pQInfo->pSidSet->starterPos[pQInfo->subgroupIdx + 1]; + + assert(0); +// ret = doMergeMetersResultsToGroupRes(pQInfo, pQuery, pRuntimeEnv, pQInfo->pTableDataInfo, start, end); + if (ret < 0) { // not enough disk space to save the data into disk + return -1; + } + + pQInfo->subgroupIdx += 1; + + // this group generates at least one result, return results + if (ret > 0) { + break; + } + + assert(pQInfo->numOfGroupResultPages == 0); + dTrace("QInfo:%p no result in group %d, continue", GET_QINFO_ADDR(pQuery), pQInfo->subgroupIdx - 1); + } + + dTrace("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%lldms", GET_QINFO_ADDR(pQuery), + pQInfo->subgroupIdx - 1, pQInfo->pSidSet->numOfSubSet, taosGetTimestampMs() - st); + + return TSDB_CODE_SUCCESS; +} + +void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { + if (pQInfo->offset == pQInfo->numOfGroupResultPages) { + pQInfo->numOfGroupResultPages = 0; + + // current results of group has been sent to client, try next group + if (mergeMetersResultToOneGroups(pQInfo) != TSDB_CODE_SUCCESS) { + return; // failed to save data in the disk + } + + // set current query completed + if (pQInfo->numOfGroupResultPages == 0 && pQInfo->subgroupIdx == pQInfo->pSidSet->numOfSubSet) { + pQInfo->tableIndex = pQInfo->pSidSet->numOfSids; + return; + } + } + + SQueryRuntimeEnv * pRuntimeEnv = &pQInfo->runtimeEnv; + SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; + + int32_t id = getGroupResultId(pQInfo->subgroupIdx - 1); + SIDList list = getDataBufPagesIdList(pResultBuf, pQInfo->offset + id); + + int32_t total = 0; + for (int32_t i = 0; i < list.size; ++i) { + tFilePage *pData = getResultBufferPageById(pResultBuf, list.pData[i]); + total += pData->numOfElems; + } + + pQuery->sdata[0]->num = total; + + int32_t offset = 0; + for (int32_t num = 0; num < list.size; ++num) { + tFilePage *pData = getResultBufferPageById(pResultBuf, list.pData[num]); + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; + char * pDest = pQuery->sdata[i]->data; + + memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pData->numOfElems, + bytes * pData->numOfElems); + } + + offset += pData->numOfElems; + } + + assert(pQuery->rec.pointsRead == 0); + + pQuery->rec.pointsRead += pQuery->sdata[0]->num; + pQInfo->offset += 1; +} + +int64_t getNumOfResultWindowRes(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindowRes) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + int64_t maxOutput = 0; + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; + + /* + * ts, tag, tagprj function can not decide the output number of current query + * the number of output result is decided by main output + */ + if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ) { + continue; + } + + SResultInfo *pResultInfo = &pWindowRes->resultInfo[j]; + if (pResultInfo != NULL && maxOutput < pResultInfo->numOfRes) { + maxOutput = pResultInfo->numOfRes; + } + } + + return maxOutput; +} + +int32_t doMergeMetersResultsToGroupRes(SQInfo *pQInfo, STableDataInfo *pTableDataInfo, int32_t start, int32_t end) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + + tFilePage ** buffer = (tFilePage **)pQuery->sdata; + int32_t * posList = calloc((end - start), sizeof(int32_t)); + STableDataInfo **pTableList = malloc(POINTER_BYTES * (end - start)); + + // todo opt for the case of one table per group + int32_t numOfMeters = 0; + for (int32_t i = start; i < end; ++i) { + int32_t sid = pTableDataInfo[i].pTableQInfo->sid; + + SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, sid); + if (list.size > 0 && pTableDataInfo[i].pTableQInfo->windowResInfo.size > 0) { + pTableList[numOfMeters] = &pTableDataInfo[i]; + numOfMeters += 1; + } + } + + if (numOfMeters == 0) { + tfree(posList); + tfree(pTableList); + + assert(pQInfo->numOfGroupResultPages == 0); + return 0; + } + + SCompSupporter cs = {pTableList, posList, pQInfo}; + + SLoserTreeInfo *pTree = NULL; + tLoserTreeCreate(&pTree, numOfMeters, &cs, tableResultComparFn); + + SResultInfo *pResultInfo = calloc(pQuery->numOfOutputCols, sizeof(SResultInfo)); + setWindowResultInfo(pResultInfo, pQuery, pRuntimeEnv->stableQuery); + + resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo); + int64_t lastTimestamp = -1; + + int64_t startt = taosGetTimestampMs(); + + while (1) { + int32_t pos = pTree->pNode[0].index; + + SWindowResInfo *pWindowResInfo = &pTableList[pos]->pTableQInfo->windowResInfo; + SWindowResult * pWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]); + + char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes); + TSKEY ts = GET_INT64_VAL(b); + + assert(ts == pWindowRes->window.skey); + int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pWindowRes); + if (num <= 0) { + cs.position[pos] += 1; + + if (cs.position[pos] >= pWindowResInfo->size) { + cs.position[pos] = -1; + + // all input sources are exhausted + if (--numOfMeters == 0) { + break; + } + } + } else { + if (ts == lastTimestamp) { // merge with the last one + doMerge(pRuntimeEnv, ts, pWindowRes, true); + } else { // copy data to disk buffer + assert(0); +// if (buffer[0]->numOfElems == pQuery->pointsToRead) { +// if (flushFromResultBuf(pQInfo) != TSDB_CODE_SUCCESS) { +// return -1; +// } + +// resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo); +// } + + doMerge(pRuntimeEnv, ts, pWindowRes, false); + buffer[0]->numOfElems += 1; + } + + lastTimestamp = ts; + + cs.position[pos] += 1; + if (cs.position[pos] >= pWindowResInfo->size) { + cs.position[pos] = -1; + + // all input sources are exhausted + if (--numOfMeters == 0) { + break; + } + } + } + + tLoserTreeAdjust(pTree, pos + pTree->numOfEntries); + } + + if (buffer[0]->numOfElems != 0) { // there are data in buffer + if (flushFromResultBuf(pQInfo) != TSDB_CODE_SUCCESS) { + // dError("QInfo:%p failed to flush data into temp file, abort query", GET_QINFO_ADDR(pQuery), + // pQInfo->extBufFile); + tfree(pTree); + tfree(pTableList); + tfree(posList); + tfree(pResultInfo); + + return -1; + } + } + + int64_t endt = taosGetTimestampMs(); + +#ifdef _DEBUG_VIEW + displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->len); +#endif + + dTrace("QInfo:%p result merge completed, elapsed time:%" PRId64 " ms", GET_QINFO_ADDR(pQuery), endt - startt); + tfree(pTree); + tfree(pTableList); + tfree(posList); + + pQInfo->offset = 0; + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + tfree(pResultInfo[i].interResultBuf); + } + + tfree(pResultInfo); + return pQInfo->numOfGroupResultPages; +} + +int32_t flushFromResultBuf(SQInfo *pQInfo) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery* pQuery = pRuntimeEnv->pQuery; + + SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; + int32_t capacity = (DEFAULT_INTERN_BUF_SIZE - sizeof(tFilePage)) / pQuery->rowSize; + + // the base value for group result, since the maximum number of table for each vnode will not exceed 100,000. + int32_t pageId = -1; + + int32_t remain = pQuery->sdata[0]->num; + int32_t offset = 0; + + while (remain > 0) { + int32_t r = remain; + if (r > capacity) { + r = capacity; + } + + int32_t id = getGroupResultId(pQInfo->subgroupIdx) + pQInfo->numOfGroupResultPages; + tFilePage *buf = getNewDataBuf(pResultBuf, id, &pageId); + + // pagewise copy to dest buffer + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; + buf->numOfElems = r; + + memcpy(buf->data + pRuntimeEnv->offset[i] * buf->numOfElems, ((char *)pQuery->sdata[i]->data) + offset * bytes, + buf->numOfElems * bytes); + } + + offset += r; + remain -= r; + } + + pQInfo->numOfGroupResultPages += 1; + return TSDB_CODE_SUCCESS; +} + +void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pResultInfo) { + for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + pCtx[k].aOutputBuf = pQuery->sdata[k]->data - pCtx[k].outputBytes; + pCtx[k].size = 1; + pCtx[k].startOffset = 0; + pCtx[k].resultInfo = &pResultInfo[k]; + + pQuery->sdata[k]->num = 0; + } +} + +void setMeterDataInfo(STableDataInfo *pTableDataInfo, void *pMeterObj, int32_t meterIdx, int32_t groupId) { + pTableDataInfo->pMeterObj = pMeterObj; + pTableDataInfo->groupIdx = groupId; + pTableDataInfo->tableIndex = meterIdx; +} + +static void doDisableFunctsForSupplementaryScan(SQuery *pQuery, SWindowResInfo *pWindowResInfo, int32_t order) { + for (int32_t i = 0; i < pWindowResInfo->size; ++i) { + SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, i); + if (!pStatus->closed) { + continue; + } + + SWindowResult *buf = getWindowResult(pWindowResInfo, i); + + // open/close the specified query for each group result + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + int32_t functId = pQuery->pSelectExpr[j].pBase.functionId; + + if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSQL_SO_DESC) || + ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSQL_SO_ASC)) { + buf->resultInfo[j].complete = false; + } else if (functId != TSDB_FUNC_TS && functId != TSDB_FUNC_TAG) { + buf->resultInfo[j].complete = true; + } + } + } +} + +void disableFunctForTableSuppleScan(SQueryRuntimeEnv *pRuntimeEnv, int32_t order) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + // group by normal columns and interval query on normal table + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + pRuntimeEnv->pCtx[i].order = (pRuntimeEnv->pCtx[i].order) ^ 1u; + } + + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { + doDisableFunctsForSupplementaryScan(pQuery, pWindowResInfo, order); + } else { // for simple result of table query, + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + int32_t functId = pQuery->pSelectExpr[j].pBase.functionId; + SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[j]; + + if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSQL_SO_DESC) || + ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSQL_SO_ASC)) { + pCtx->resultInfo->complete = false; + } else if (functId != TSDB_FUNC_TS && functId != TSDB_FUNC_TAG) { + pCtx->resultInfo->complete = true; + } + } + } + + pQuery->order.order = pQuery->order.order ^ 1u; +} + +void disableFunctForSuppleScan(SQInfo *pQInfo, int32_t order) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + pRuntimeEnv->pCtx[i].order = (pRuntimeEnv->pCtx[i].order) ^ 1u; + } + + if (isIntervalQuery(pQuery)) { + size_t numOfTables = taosHashGetSize(pQInfo->pTableList); + + for (int32_t i = 0; i < numOfTables; ++i) { + STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; + SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; + + doDisableFunctsForSupplementaryScan(pQuery, pWindowResInfo, order); + } + } else { + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + doDisableFunctsForSupplementaryScan(pQuery, pWindowResInfo, order); + } + + pQuery->order.order = (pQuery->order.order) ^ 1u; +} + +void enableFunctForMasterScan(SQueryRuntimeEnv *pRuntimeEnv, int32_t order) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + pRuntimeEnv->pCtx[i].order = (pRuntimeEnv->pCtx[i].order) ^ 1u; + } + + pQuery->order.order = (pQuery->order.order) ^ 1u; +} + +void createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, SPosInfo *posInfo) { + int32_t numOfCols = pQuery->numOfOutputCols; + + pResultRow->resultInfo = calloc((size_t)numOfCols, sizeof(SResultInfo)); + pResultRow->pos = *posInfo; + + // set the intermediate result output buffer + setWindowResultInfo(pResultRow->resultInfo, pQuery, isSTableQuery); +} + +void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; +// int32_t rows = pRuntimeEnv->pTabObj->pointsPerFileBlock; + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; + pCtx->aOutputBuf = pQuery->sdata[i]->data; + + /* + * set the output buffer information and intermediate buffer + * not all queries require the interResultBuf, such as COUNT/TAGPRJ/PRJ/TAG etc. + */ + resetResultInfo(&pRuntimeEnv->resultInfo[i]); + pCtx->resultInfo = &pRuntimeEnv->resultInfo[i]; + + // set the timestamp output buffer for top/bottom/diff query + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { + pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; + } + + assert(0); +// memset(pQuery->sdata[i]->data, 0, (size_t)pQuery->pSelectExpr[i].resBytes * rows); + } + + initCtxOutputBuf(pRuntimeEnv); +} + +void forwardCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, int64_t output) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + // reset the execution contexts + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; + assert(functionId != TSDB_FUNC_DIFF); + + // set next output position + if (IS_OUTER_FORWARD(aAggs[functionId].nStatus)) { + pRuntimeEnv->pCtx[j].aOutputBuf += pRuntimeEnv->pCtx[j].outputBytes * output; + } + + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { + /* + * NOTE: for top/bottom query, the value of first column of output (timestamp) are assigned + * in the procedure of top/bottom routine + * the output buffer in top/bottom routine is ptsOutputBuf, so we need to forward the output buffer + * + * diff function is handled in multi-output function + */ + pRuntimeEnv->pCtx[j].ptsOutputBuf += TSDB_KEYSIZE * output; + } + + resetResultInfo(pRuntimeEnv->pCtx[j].resultInfo); + } +} + +void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; + pRuntimeEnv->pCtx[j].currentStage = 0; + + aAggs[functionId].init(&pRuntimeEnv->pCtx[j]); + } +} + +void doSkipResults(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; + if (pQuery->rec.pointsRead == 0 || pQuery->limit.offset == 0) { + return; + } + + if (pQuery->rec.pointsRead <= pQuery->limit.offset) { + pQuery->limit.offset -= pQuery->rec.pointsRead; + + pQuery->rec.pointsRead = 0; +// pQuery->pointsOffset = pQuery->rec.pointsToRead; // clear all data in result buffer + + resetCtxOutputBuf(pRuntimeEnv); + + // clear the buffer is full flag if exists + pQuery->status &= (~QUERY_RESBUF_FULL); + } else { + int32_t numOfSkip = (int32_t)pQuery->limit.offset; + pQuery->rec.pointsRead -= numOfSkip; + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; + assert(0); +// memmove(pQuery->sdata[i]->data, pQuery->sdata[i]->data + bytes * numOfSkip, pQuery->pointsRead * bytes); + pRuntimeEnv->pCtx[i].aOutputBuf += bytes * numOfSkip; + + if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { + pRuntimeEnv->pCtx[i].ptsOutputBuf += TSDB_KEYSIZE * numOfSkip; + } + } + + pQuery->limit.offset = 0; + } +} + +typedef struct SQueryStatus { + int8_t overStatus; + TSKEY lastKey; + STSCursor cur; +} SQueryStatus; + +// todo refactor +static void queryStatusSave(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatus *pStatus) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + pStatus->overStatus = pQuery->status; + pStatus->lastKey = pQuery->lastKey; + + pStatus->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); // save the cursor + + if (pRuntimeEnv->pTSBuf) { + pRuntimeEnv->pTSBuf->cur.order ^= 1u; + tsBufNextPos(pRuntimeEnv->pTSBuf); + } + + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); + + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + pQuery->lastKey = pQuery->window.skey; +} + +static void queryStatusRestore(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatus *pStatus) { + SQuery *pQuery = pRuntimeEnv->pQuery; + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + + pQuery->lastKey = pStatus->lastKey; + pQuery->status = pStatus->overStatus; + + tsBufSetCursor(pRuntimeEnv->pTSBuf, &pStatus->cur); +} + +static void doSingleMeterSupplementScan(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery * pQuery = pRuntimeEnv->pQuery; + SQueryStatus qStatus = {0}; + + if (!needSupplementaryScan(pQuery)) { + return; + } + + dTrace("QInfo:%p start to supp scan", GET_QINFO_ADDR(pQuery)); + SET_SUPPLEMENT_SCAN_FLAG(pRuntimeEnv); + + // close necessary function execution during supplementary scan + disableFunctForTableSuppleScan(pRuntimeEnv, pQuery->order.order); + queryStatusSave(pRuntimeEnv, &qStatus); + + STimeWindow w = {.skey = pQuery->window.skey, .ekey = pQuery->window.ekey}; + + // reverse scan from current position + tsdbpos_t current = tsdbDataBlockTell(pRuntimeEnv->pQueryHandle); + tsdbResetQuery(pRuntimeEnv->pQueryHandle, &w, current, pQuery->order.order); + + doScanAllDataBlocks(pRuntimeEnv); + + queryStatusRestore(pRuntimeEnv, &qStatus); + enableFunctForMasterScan(pRuntimeEnv, pQuery->order.order); + SET_MASTER_SCAN_FLAG(pRuntimeEnv); +} + +void setQueryStatus(SQuery *pQuery, int8_t status) { + if (status == QUERY_NOT_COMPLETED) { + pQuery->status = status; + } else { + // QUERY_NOT_COMPLETED is not compatible with any other status, so clear its position first + pQuery->status &= (~QUERY_NOT_COMPLETED); + pQuery->status |= status; + } +} + +bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; + bool toContinue = false; + + if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { + // for each group result, call the finalize function for each column + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + + for (int32_t i = 0; i < pWindowResInfo->size; ++i) { + SWindowResult *pResult = getWindowResult(pWindowResInfo, i); + if (!pResult->status.closed) { + continue; + } + + setWindowResOutputBuf(pRuntimeEnv, pResult); + + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + int16_t functId = pQuery->pSelectExpr[j].pBase.functionId; + if (functId == TSDB_FUNC_TS) { + continue; + } + + aAggs[functId].xNextStep(&pRuntimeEnv->pCtx[j]); + SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + + toContinue |= (!pResInfo->complete); + } + } + } else { + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + int16_t functId = pQuery->pSelectExpr[j].pBase.functionId; + if (functId == TSDB_FUNC_TS) { + continue; + } + + aAggs[functId].xNextStep(&pRuntimeEnv->pCtx[j]); + SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + + toContinue |= (!pResInfo->complete); + } + } + + return toContinue; +} + +void vnodeScanAllData(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); + + // store the start query position + void *pos = tsdbDataBlockTell(pRuntimeEnv->pQueryHandle); + + int64_t skey = pQuery->lastKey; + int32_t status = pQuery->status; + int32_t activeSlot = pRuntimeEnv->windowResInfo.curIndex; + + SET_MASTER_SCAN_FLAG(pRuntimeEnv); + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + + while (1) { + doScanAllDataBlocks(pRuntimeEnv); + + if (!needScanDataBlocksAgain(pRuntimeEnv)) { + // restore the status + if (pRuntimeEnv->scanFlag == REPEAT_SCAN) { + pQuery->status = status; + } + + break; + } + + /* + * set the correct start position, and load the corresponding block in buffer for next + * round scan all data blocks. + */ + int32_t ret = tsdbDataBlockSeek(pRuntimeEnv->pQueryHandle, pos); + + status = pQuery->status; + pRuntimeEnv->windowResInfo.curIndex = activeSlot; + + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); + pRuntimeEnv->scanFlag = REPEAT_SCAN; + + /* check if query is killed or not */ + if (isQueryKilled(pQuery)) { + setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); + return; + } + } + + // no need to set the end key + TSKEY lkey = pQuery->lastKey; + TSKEY ekey = pQuery->window.ekey; + + pQuery->window.skey = skey; + pQuery->window.ekey = pQuery->lastKey - step; + tsdbpos_t current = tsdbDataBlockTell(pRuntimeEnv->pQueryHandle); + + doSingleMeterSupplementScan(pRuntimeEnv); + + // update the pQuery->window.skey and pQuery->window.ekey to limit the scan scope of sliding query during supplementary scan + pQuery->lastKey = lkey; + pQuery->window.ekey = ekey; + + STimeWindow win = {.skey = pQuery->window.skey, .ekey = pQuery->window.ekey}; + tsdbResetQuery(pRuntimeEnv->pQueryHandle, &win, current, pQuery->order.order); + tsdbNextDataBlock(pRuntimeEnv->pQueryHandle); +} + +void doFinalizeResult(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { + // for each group result, call the finalize function for each column + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + closeAllTimeWindow(pWindowResInfo); + } + + for (int32_t i = 0; i < pWindowResInfo->size; ++i) { + SWindowResult *buf = &pWindowResInfo->pResult[i]; + if (!isWindowResClosed(pWindowResInfo, i)) { + continue; + } + + setWindowResOutputBuf(pRuntimeEnv, buf); + + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + aAggs[pQuery->pSelectExpr[j].pBase.functionId].xFinalize(&pRuntimeEnv->pCtx[j]); + } + + /* + * set the number of output results for group by normal columns, the number of output rows usually is 1 except + * the top and bottom query + */ + buf->numOfRows = getNumOfResult(pRuntimeEnv); + } + + } else { + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + aAggs[pQuery->pSelectExpr[j].pBase.functionId].xFinalize(&pRuntimeEnv->pCtx[j]); + } + } +} + +static bool hasMainOutput(SQuery *pQuery) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + + if (functionId != TSDB_FUNC_TS && functionId != TSDB_FUNC_TAG && functionId != TSDB_FUNC_TAGPRJ) { + return true; + } + } + + return false; +} + +STableQueryInfo *createMeterQueryInfo(SQInfo *pQInfo, int32_t sid, TSKEY skey, TSKEY ekey) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + + STableQueryInfo *pTableQueryInfo = calloc(1, sizeof(STableQueryInfo)); + + pTableQueryInfo->win = (STimeWindow) {.skey = skey, .ekey = ekey,}; + pTableQueryInfo->lastKey = skey; + + pTableQueryInfo->sid = sid; + pTableQueryInfo->cur.vnodeIndex = -1; + + initWindowResInfo(&pTableQueryInfo->windowResInfo, pRuntimeEnv, 100, 100, TSDB_DATA_TYPE_INT); + return pTableQueryInfo; +} + +void destroyMeterQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols) { + if (pTableQueryInfo == NULL) { + return; + } + + cleanupTimeWindowInfo(&pTableQueryInfo->windowResInfo, numOfCols); + free(pTableQueryInfo); +} + +void changeMeterQueryInfoForSuppleQuery(SQuery *pQuery, STableQueryInfo *pTableQueryInfo, TSKEY skey, TSKEY ekey) { + if (pTableQueryInfo == NULL) { + return; + } + + // order has change already! + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + if (!QUERY_IS_ASC_QUERY(pQuery)) { + assert(pTableQueryInfo->win.ekey >= pTableQueryInfo->lastKey + step); + } else { + assert(pTableQueryInfo->win.ekey <= pTableQueryInfo->lastKey + step); + } + + pTableQueryInfo->win.ekey = pTableQueryInfo->lastKey + step; + + SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); + pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; + + pTableQueryInfo->cur.order = pTableQueryInfo->cur.order ^ 1u; + pTableQueryInfo->cur.vnodeIndex = -1; +} + +void restoreIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *pTableQueryInfo) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + pQuery->window.skey = pTableQueryInfo->win.skey; + pQuery->window.ekey = pTableQueryInfo->win.ekey; + pQuery->lastKey = pTableQueryInfo->lastKey; + + assert(((pQuery->lastKey >= pQuery->window.skey) && QUERY_IS_ASC_QUERY(pQuery)) || + ((pQuery->lastKey <= pQuery->window.skey) && !QUERY_IS_ASC_QUERY(pQuery))); +} + +/** + * set output buffer for different group + * @param pRuntimeEnv + * @param pDataBlockInfoEx + */ +void setExecutionContext(SQInfo *pQInfo, STableQueryInfo *pTableQueryInfo, int32_t meterIdx, + int32_t groupIdx, TSKEY nextKey) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SWindowResInfo * pWindowResInfo = &pRuntimeEnv->windowResInfo; + int32_t GROUPRESULTID = 1; + + SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIdx, sizeof(groupIdx)); + if (pWindowRes == NULL) { + return; + } + + /* + * not assign result buffer yet, add new result buffer + * all group belong to one result set, and each group result has different group id so set the id to be one + */ + if (pWindowRes->pos.pageId == -1) { + if (addNewWindowResultBuf(pWindowRes, pRuntimeEnv->pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage) != + TSDB_CODE_SUCCESS) { + return; + } + } + + setWindowResOutputBuf(pRuntimeEnv, pWindowRes); + initCtxOutputBuf(pRuntimeEnv); + + pTableQueryInfo->lastKey = nextKey; + setAdditionalInfo(pQInfo, meterIdx, pTableQueryInfo); +} + +static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + // Note: pResult->pos[i]->numOfElems == 0, there is only fixed number of results for each group + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; + pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult); + + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { + pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; + } + + /* + * set the output buffer information and intermediate buffer + * not all queries require the interResultBuf, such as COUNT + */ + pCtx->resultInfo = &pResult->resultInfo[i]; + + // set super table query flag + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + pResInfo->superTableQ = pRuntimeEnv->stableQuery; + } +} + +int32_t setAdditionalInfo(SQInfo *pQInfo, int32_t meterIdx, STableQueryInfo *pTableQueryInfo) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + assert(pTableQueryInfo->lastKey > 0); + +// vnodeSetTagValueInParam(pQInfo->pSidSet, pRuntimeEnv, pQInfo->pMeterSidExtInfo[meterIdx]); + + // both the master and supplement scan needs to set the correct ts comp start position + if (pRuntimeEnv->pTSBuf != NULL) { + if (pTableQueryInfo->cur.vnodeIndex == -1) { + pTableQueryInfo->tag = pRuntimeEnv->pCtx[0].tag.i64Key; + + tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, pTableQueryInfo->tag); + + // keep the cursor info of current meter + pTableQueryInfo->cur = pRuntimeEnv->pTSBuf->cur; + } else { + tsBufSetCursor(pRuntimeEnv->pTSBuf, &pTableQueryInfo->cur); + } + } + + return 0; +} + +/* + * There are two cases to handle: + * + * 1. Query range is not set yet (queryRangeSet = 0). we need to set the query range info, including pQuery->lastKey, + * pQuery->window.skey, and pQuery->eKey. + * 2. Query range is set and query is in progress. There may be another result with the same query ranges to be + * merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there + * is a previous result generated or not. + */ +void setIntervalQueryRange(STableQueryInfo *pTableQueryInfo, SQInfo *pQInfo, TSKEY key) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + + if (pTableQueryInfo->queryRangeSet) { + pQuery->lastKey = key; + pTableQueryInfo->lastKey = key; + } else { + pQuery->window.skey = key; + STimeWindow win = {.skey = key, pQuery->window.ekey}; + + // for too small query range, no data in this interval. + if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.ekey < pQuery->window.skey)) || + (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.skey < pQuery->window.ekey))) { + return; + } + + /** + * In handling the both ascending and descending order super table query, we need to find the first qualified + * timestamp of this table, and then set the first qualified start timestamp. + * In ascending query, key is the first qualified timestamp. However, in the descending order query, additional + * operations involve. + */ + TSKEY skey1, ekey1; + STimeWindow w = {0}; + SWindowResInfo *pWindowResInfo = &pTableQueryInfo->windowResInfo; + + doGetAlignedIntervalQueryRangeImpl(pQuery, win.skey, win.skey, win.ekey, &skey1, &ekey1, &w); + pWindowResInfo->startTime = pQuery->window.skey; // windowSKey may be 0 in case of 1970 timestamp + + if (pWindowResInfo->prevSKey == 0) { + if (QUERY_IS_ASC_QUERY(pQuery)) { + pWindowResInfo->prevSKey = w.skey; + } else { + assert(win.ekey == pQuery->window.skey); + pWindowResInfo->prevSKey = w.skey; + } + } + + pTableQueryInfo->queryRangeSet = 1; + pTableQueryInfo->lastKey = pQuery->window.skey; + pTableQueryInfo->win.skey = pQuery->window.skey; + + pQuery->lastKey = pQuery->window.skey; + } +} + +bool requireTimestamp(SQuery *pQuery) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; i++) { + int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_NEED_TS) != 0) { + return true; + } + } + return false; +} + +bool needPrimaryTimestampCol(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo) { + /* + * 1. if skey or ekey locates in this block, we need to load the timestamp column to decide the precise position + * 2. if there are top/bottom, first_dst/last_dst functions, we need to load timestamp column in any cases; + */ + STimeWindow *w = &pDataBlockInfo->window; + bool loadPrimaryTS = (pQuery->lastKey >= w->skey && pQuery->lastKey <= w->ekey) || + (pQuery->window.ekey >= w->skey && pQuery->window.ekey <= w->ekey) || requireTimestamp(pQuery); + + return loadPrimaryTS; +} + +bool onDemandLoadDatablock(SQuery *pQuery, int16_t queryRangeSet) { + return (pQuery->intervalTime == 0) || ((queryRangeSet == 1) && (isIntervalQuery(pQuery))); +} + +static int32_t getNumOfSubset(SQInfo *pQInfo) { + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + int32_t totalSubset = 0; + if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || (isIntervalQuery(pQuery))) { + totalSubset = numOfClosedTimeWindow(&pQInfo->runtimeEnv.windowResInfo); + } else { + totalSubset = pQInfo->pSidSet->numOfSubSet; + } + + return totalSubset; +} + +static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orderType) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + + int32_t numOfResult = 0; + int32_t startIdx = 0; + int32_t step = -1; + + dTrace("QInfo:%p start to copy data from windowResInfo to pQuery buf", GET_QINFO_ADDR(pQuery)); + int32_t totalSubset = getNumOfSubset(pQInfo); + + if (orderType == TSQL_SO_ASC) { + startIdx = pQInfo->subgroupIdx; + step = 1; + } else { // desc order copy all data + startIdx = totalSubset - pQInfo->subgroupIdx - 1; + step = -1; + } + + for (int32_t i = startIdx; (i < totalSubset) && (i >= 0); i += step) { + if (result[i].numOfRows == 0) { + pQInfo->offset = 0; + pQInfo->subgroupIdx += 1; + continue; + } + + assert(result[i].numOfRows >= 0 && pQInfo->offset <= 1); + + int32_t numOfRowsToCopy = result[i].numOfRows - pQInfo->offset; + int32_t oldOffset = pQInfo->offset; + assert(0); + + /* + * current output space is not enough to keep all the result data of this group, only copy partial results + * to SQuery object's result buffer + */ +// if (numOfRowsToCopy > pQuery->pointsToRead - numOfResult) { +// numOfRowsToCopy = pQuery->pointsToRead - numOfResult; +// pQInfo->offset += numOfRowsToCopy; +// } else { +// pQInfo->offset = 0; +// pQInfo->subgroupIdx += 1; +// } + + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + int32_t size = pRuntimeEnv->pCtx[j].outputBytes; + + char *out = pQuery->sdata[j]->data + numOfResult * size; + char *in = getPosInResultPage(pRuntimeEnv, j, &result[i]); + memcpy(out, in + oldOffset * size, size * numOfRowsToCopy); + } + + numOfResult += numOfRowsToCopy; + assert(0); +// if (numOfResult == pQuery->rec.pointsToRead) { +// break; +// } + } + + dTrace("QInfo:%p copy data to SQuery buf completed", GET_QINFO_ADDR(pQuery)); + +#ifdef _DEBUG_VIEW + displayInterResult(pQuery->sdata, pQuery, numOfResult); +#endif + return numOfResult; +} + +/** + * copyFromWindowResToSData support copy data in ascending/descending order + * For interval query of both super table and table, copy the data in ascending order, since the output results are + * ordered in SWindowResutl already. While handling the group by query for both table and super table, + * all group result are completed already. + * + * @param pQInfo + * @param result + */ +void copyFromWindowResToSData(SQInfo *pQInfo, SWindowResult *result) { + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + int32_t orderType = (pQuery->pGroupbyExpr != NULL) ? pQuery->pGroupbyExpr->orderType : TSQL_SO_ASC; + int32_t numOfResult = doCopyToSData(pQInfo, result, orderType); + + pQuery->rec.pointsRead += numOfResult; +// assert(pQuery->rec.pointsRead <= pQuery->pointsToRead); +} + +static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv, STableDataInfo *pTableDataInfo) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + // update the number of result for each, only update the number of rows for the corresponding window result. + if (pQuery->intervalTime == 0) { + int32_t g = pTableDataInfo->groupIdx; + assert(pRuntimeEnv->windowResInfo.size > 0); + + SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, (char *)&g, sizeof(g)); + if (pWindowRes->numOfRows == 0) { + pWindowRes->numOfRows = getNumOfResult(pRuntimeEnv); + } + } +} + +void stableApplyFunctionsOnBlock_(SQInfo *pQInfo, STableDataInfo *pTableDataInfo, + SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, SArray* pDataBlock, + __block_search_fn_t searchFn) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + STableQueryInfo * pTableQueryInfo = pTableDataInfo->pTableQInfo; + SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; + + int32_t numOfRes = 0; + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) { +// numOfRes = rowwiseApplyAllFunctions(pRuntimeEnv, &forwardStep, pFields, pDataBlockInfo, pWindowResInfo); + } else { + numOfRes = blockwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); + } + + assert(numOfRes >= 0); + + updateWindowResNumOfRes(pRuntimeEnv, pTableDataInfo); + updatelastkey(pQuery, pTableQueryInfo); +} + +// we need to split the refstatsult into different packages. +int32_t vnodeGetResultSize(void *thandle, int32_t *numOfRows) { + SQInfo *pQInfo = (SQInfo *)thandle; + SQuery *pQuery = &pQInfo->runtimeEnv.pQuery; + + /* + * get the file size and set the numOfRows to be the file size, since for tsComp query, + * the returned row size is equalled to 1 + * + * TODO handle the case that the file is too large to send back one time + */ + if (isTSCompQuery(pQuery) && (*numOfRows) > 0) { + struct stat fstat; + if (stat(pQuery->sdata[0]->data, &fstat) == 0) { + *numOfRows = fstat.st_size; + return fstat.st_size; + } else { + dError("QInfo:%p failed to get file info, path:%s, reason:%s", pQInfo, pQuery->sdata[0]->data, strerror(errno)); + return 0; + } + } else { + return pQuery->rowSize * (*numOfRows); + } +} + +int64_t vnodeGetOffsetVal(void *thandle) { + SQInfo *pQInfo = (SQInfo *)thandle; + return pQInfo->runtimeEnv.pQuery->limit.offset; +} + +bool vnodeHasRemainResults(void *handle) { + SQInfo *pQInfo = (SQInfo *)handle; + + if (pQInfo == NULL || pQInfo->runtimeEnv.pQuery->interpoType == TSDB_INTERPO_NONE) { + return false; + } + + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + + SInterpolationInfo *pInterpoInfo = &pRuntimeEnv->interpoInfo; + if (pQuery->limit.limit > 0 && pQInfo->rec.pointsRead >= pQuery->limit.limit) { + return false; + } + + int32_t remain = taosNumOfRemainPoints(pInterpoInfo); + if (remain > 0) { + return true; + } else { + if (pRuntimeEnv->pInterpoBuf == NULL) { + return false; + } + + // query has completed + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)) { + TSKEY ekey = taosGetRevisedEndKey(pQuery->window.ekey, pQuery->order.order, pQuery->intervalTime, + pQuery->slidingTimeUnit, pQuery->precision); +// int32_t numOfTotal = taosGetNumOfResultWithInterpo(pInterpoInfo, (TSKEY *)pRuntimeEnv->pInterpoBuf[0]->data, +// remain, pQuery->intervalTime, ekey, pQuery->pointsToRead); +// return numOfTotal > 0; + assert(0); + return false; + } + + return false; + } +} + +static int32_t resultInterpolate(SQInfo *pQInfo, tFilePage **data, tFilePage **pDataSrc, int32_t numOfRows, + int32_t outputRows) { +#if 0 + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery *pQuery = &pRuntimeEnv->pQuery; + + assert(pRuntimeEnv->pCtx[0].outputBytes == TSDB_KEYSIZE); + + // build support structure for performing interpolation + SSchema *pSchema = calloc(1, sizeof(SSchema) * pQuery->numOfOutputCols); + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + pSchema[i].bytes = pRuntimeEnv->pCtx[i].outputBytes; + pSchema[i].type = pQuery->pSelectExpr[i].resType; + } + +// SColumnModel *pModel = createColumnModel(pSchema, pQuery->numOfOutputCols, pQuery->pointsToRead); + + char * srcData[TSDB_MAX_COLUMNS] = {0}; + int32_t functions[TSDB_MAX_COLUMNS] = {0}; + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + srcData[i] = pDataSrc[i]->data; + functions[i] = pQuery->pSelectExpr[i].pBase.functionId; + } + + assert(0); +// int32_t numOfRes = taosDoInterpoResult(&pRuntimeEnv->interpoInfo, pQuery->interpoType, data, numOfRows, outputRows, +// pQuery->intervalTime, (int64_t *)pDataSrc[0]->data, pModel, srcData, +// pQuery->defaultVal, functions, pRuntimeEnv->pTabObj->pointsPerFileBlock); + + destroyColumnModel(pModel); + free(pSchema); +#endif + return 0; +} + +static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data) { +#if 0 + SMeterObj *pObj = pQInfo->pObj; + SQuery * pQuery = &pQInfo->query; + + int tnumOfRows = vnodeList[pObj->vnode].cfg.rowsInFileBlock; + + // for metric query, bufIndex always be 0. + for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { // pQInfo->bufIndex == 0 + int32_t bytes = pQuery->pSelectExpr[col].resBytes; + + memmove(data, pQuery->sdata[col]->data, bytes * numOfRows); + data += bytes * numOfRows; + } +#endif +} + +/** + * Copy the result data/file to output message buffer. + * If the result is in file format, read file from disk and copy to output buffer, compression is not involved since + * data in file is already compressed. + * In case of other result in buffer, compress the result before copy once the tsComressMsg is set. + * + * @param handle + * @param data + * @param numOfRows the number of rows that are not returned in current retrieve + * @return + */ +int32_t vnodeCopyQueryResultToMsg(void *handle, char *data, int32_t numOfRows) { + SQInfo *pQInfo = (SQInfo *)handle; + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + assert(pQuery->pSelectExpr != NULL && pQuery->numOfOutputCols > 0); + + // load data from file to msg buffer + if (isTSCompQuery(pQuery)) { + int32_t fd = open(pQuery->sdata[0]->data, O_RDONLY, 0666); + + // make sure file exist + if (FD_VALID(fd)) { + size_t s = lseek(fd, 0, SEEK_END); + dTrace("QInfo:%p ts comp data return, file:%s, size:%zu", pQInfo, pQuery->sdata[0]->data, s); + + lseek(fd, 0, SEEK_SET); + read(fd, data, s); + close(fd); + + unlink(pQuery->sdata[0]->data); + } else { + dError("QInfo:%p failed to open tmp file to send ts-comp data to client, path:%s, reason:%s", pQInfo, + pQuery->sdata[0]->data, strerror(errno)); + } + } else { + doCopyQueryResultToMsg(pQInfo, numOfRows, data); + } + + return numOfRows; +} + +int32_t vnodeQueryResultInterpolate(SQInfo *pQInfo, tFilePage **pDst, tFilePage **pDataSrc, int32_t numOfRows, + int32_t *numOfInterpo) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; +#if 0 + while (1) { + numOfRows = taosNumOfRemainPoints(&pRuntimeEnv->interpoInfo); + + TSKEY ekey = taosGetRevisedEndKey(pQuery->window.skey, pQuery->order.order, pQuery->intervalTime, + pQuery->slidingTimeUnit, pQuery->precision); + int32_t numOfFinalRows = taosGetNumOfResultWithInterpo(&pRuntimeEnv->interpoInfo, (TSKEY *)pDataSrc[0]->data, + numOfRows, pQuery->intervalTime, ekey, pQuery->pointsToRead); + + int32_t ret = resultInterpolate(pQInfo, pDst, pDataSrc, numOfRows, numOfFinalRows); + assert(ret == numOfFinalRows); + + /* reached the start position of according to offset value, return immediately */ + if (pQuery->limit.offset == 0) { + return ret; + } + + if (pQuery->limit.offset < ret) { + ret -= pQuery->limit.offset; + // todo !!!!there exactly number of interpo is not valid. + // todo refactor move to the beginning of buffer + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + memmove(pDst[i]->data, pDst[i]->data + pQuery->pSelectExpr[i].resBytes * pQuery->limit.offset, + ret * pQuery->pSelectExpr[i].resBytes); + } + pQuery->limit.offset = 0; + return ret; + } else { + pQuery->limit.offset -= ret; + ret = 0; + } + + if (!vnodeHasRemainResults(pQInfo)) { + return ret; + } + } +#endif + +} + +void vnodePrintQueryStatistics(SQInfo *pQInfo) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + + SQuery *pQuery = pRuntimeEnv->pQuery; +#if 0 + SQueryCostSummary *pSummary = &pRuntimeEnv->summary; + if (pRuntimeEnv->pResultBuf == NULL) { + pSummary->tmpBufferInDisk = 0; + } else { + pSummary->tmpBufferInDisk = getResBufSize(pRuntimeEnv->pResultBuf); + } + + dTrace("QInfo:%p statis: comp blocks:%d, size:%d Bytes, elapsed time:%.2f ms", pQInfo, pSummary->readCompInfo, + pSummary->totalCompInfoSize, pSummary->loadCompInfoUs / 1000.0); + + dTrace("QInfo:%p statis: field info: %d, size:%d Bytes, avg size:%.2f Bytes, elapsed time:%.2f ms", pQInfo, + pSummary->readField, pSummary->totalFieldSize, (double)pSummary->totalFieldSize / pSummary->readField, + pSummary->loadFieldUs / 1000.0); + + dTrace( + "QInfo:%p statis: file blocks:%d, size:%d Bytes, elapsed time:%.2f ms, skipped:%d, in-memory gen null:%d Bytes", + pQInfo, pSummary->readDiskBlocks, pSummary->totalBlockSize, pSummary->loadBlocksUs / 1000.0, + pSummary->skippedFileBlocks, pSummary->totalGenData); + + dTrace("QInfo:%p statis: cache blocks:%d", pQInfo, pSummary->blocksInCache, 0); + dTrace("QInfo:%p statis: temp file:%d Bytes", pQInfo, pSummary->tmpBufferInDisk); + + dTrace("QInfo:%p statis: file:%d, table:%d", pQInfo, pSummary->numOfFiles, pSummary->numOfTables); + dTrace("QInfo:%p statis: seek ops:%d", pQInfo, pSummary->numOfSeek); + + double total = pSummary->fileTimeUs + pSummary->cacheTimeUs; + double io = pSummary->loadCompInfoUs + pSummary->loadBlocksUs + pSummary->loadFieldUs; + + // todo add the intermediate result save cost!! + double computing = total - io; + + dTrace( + "QInfo:%p statis: total elapsed time:%.2f ms, file:%.2f ms(%.2f%), cache:%.2f ms(%.2f%). io:%.2f ms(%.2f%)," + "comput:%.2fms(%.2f%)", + pQInfo, total / 1000.0, pSummary->fileTimeUs / 1000.0, pSummary->fileTimeUs * 100 / total, + pSummary->cacheTimeUs / 1000.0, pSummary->cacheTimeUs * 100 / total, io / 1000.0, io * 100 / total, + computing / 1000.0, computing * 100 / total); +#endif +} + +int32_t vnodeQueryTablePrepare(SQInfo *pQInfo, void *pMeterObj, void *param) { + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + int32_t code = TSDB_CODE_SUCCESS; + + //only the successful complete requries the sem_post/over = 1 operations. + if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.skey > pQuery->window.ekey)) || + (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.ekey > pQuery->window.skey))) { + dTrace("QInfo:%p no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo, pQuery->window.skey, pQuery->window.ekey, + pQuery->order.order); + + sem_post(&pQInfo->dataReady); +// pQInfo->over = 1; + return TSDB_CODE_SUCCESS; + } + + setScanLimitationByResultBuffer(pQuery); + changeExecuteScanOrder(pQuery, false); + +// pQInfo->over = 0; + pQInfo->rec = (SResultRec) {0}; +// pQuery->pointsRead = 0; + + // dataInCache requires lastKey value + pQuery->lastKey = pQuery->window.skey; + + STsdbQueryCond cond = {0}; + cond.twindow = (STimeWindow){.skey = pQuery->window.skey, .ekey = pQuery->window.ekey}; + cond.order = pQuery->order.order; + + cond.colList = *pQuery->colList; + SArray *sa = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(sa, &pMeterObj); + + SArray *cols = taosArrayInit(pQuery->numOfCols, sizeof(pQuery->colList[0])); + for (int32_t i = 0; i < pQuery->numOfCols; ++i) { + taosArrayPush(cols, &pQuery->colList[i]); + } + + pQInfo->runtimeEnv.pQueryHandle = tsdbQueryByTableId(&cond, sa, cols); + + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + pRuntimeEnv->pQuery = pQuery; + pRuntimeEnv->pTabObj = pMeterObj; + + pRuntimeEnv->pTSBuf = param; + pRuntimeEnv->cur.vnodeIndex = -1; + if (param != NULL) { + int16_t order = (pQuery->order.order == pRuntimeEnv->pTSBuf->tsOrder) ? TSQL_SO_ASC : TSQL_SO_DESC; + tsBufSetTraverseOrder(pRuntimeEnv->pTSBuf, order); + } + + // create runtime environment + code = setupQueryRuntimeEnv(pMeterObj, pQuery, &pQInfo->runtimeEnv, NULL, pQuery->order.order, false); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + pRuntimeEnv->numOfRowsPerPage = getNumOfRowsInResultPage(pQuery, false); + if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { + int32_t rows = getInitialPageNum(pQInfo); + + code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + int16_t type = TSDB_DATA_TYPE_NULL; + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); + } else { + type = TSDB_DATA_TYPE_TIMESTAMP; + } + + initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, rows, 4096, type); + } + + /* query on single table */ + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); + + SPointInterpoSupporter interpInfo = {0}; + pointInterpSupporterInit(pQuery, &interpInfo); + + /* + * in case of last_row query without query range, we set the query timestamp to + * pMeterObj->lastKey. Otherwise, keep the initial query time range unchanged. + */ + if (isFirstLastRowQuery(pQuery) && notHasQueryTimeRange(pQuery)) { + if (!normalizeUnBoundLastRowQuery(pQInfo, &interpInfo)) { + sem_post(&pQInfo->dataReady); + pointInterpSupporterDestroy(&interpInfo); + return TSDB_CODE_SUCCESS; + } + } + + /* + * here we set the value for before and after the specified time into the + * parameter for interpolation query + */ + pointInterpSupporterSetData(pQInfo, &interpInfo); + pointInterpSupporterDestroy(&interpInfo); + + // todo move to other location + // if (!forwardQueryStartPosIfNeeded(pQInfo, pQInfo, dataInDisk, dataInCache)) { + // return TSDB_CODE_SUCCESS; + // } + + int64_t rs = taosGetIntervalStartTimestamp(pQuery->window.skey, pQuery->intervalTime, pQuery->slidingTimeUnit, + pQuery->precision); + taosInitInterpoInfo(&pRuntimeEnv->interpoInfo, pQuery->order.order, rs, 0, 0); +// allocMemForInterpo(pQInfo, pQuery, pMeterObj); + + if (!isPointInterpoQuery(pQuery)) { + // assert(pQuery->pos >= 0 && pQuery->slot >= 0); + } + + // the pQuery->window.skey is changed during normalizedFirstQueryRange, so set the newest lastkey value + pQuery->lastKey = pQuery->window.skey; + pRuntimeEnv->stableQuery = false; + + return TSDB_CODE_SUCCESS; +} + + +static bool isGroupbyEachTable(SSqlGroupbyExpr *pGroupbyExpr, tSidSet *pSidset) { + if (pGroupbyExpr == NULL || pGroupbyExpr->numOfGroupCols == 0) { + return false; + } + + for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { + SColIndexEx *pColIndex = &pGroupbyExpr->columnInfo[i]; + if (pColIndex->flag == TSDB_COL_TAG) { + assert(pSidset->numOfSids == pSidset->numOfSubSet); + return true; + } + } + + return false; +} + +static bool doCheckWithPrevQueryRange(SQuery *pQuery, TSKEY nextKey) { + if ((nextKey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (nextKey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) { + return false; + } + + return true; +} + + + +static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[i]); + if (pResInfo != NULL) { + pResInfo->complete = false; + } + } +} + +static void queryOnDataBlocks(SQInfo *pQInfo, STableDataInfo *pMeterDataInfo) { + SQueryRuntimeEnv * pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + +// SMeterObj * pTempMeter = getMeterObj(pSupporter->pMetersHashTable, pSupporter->pMeterSidExtInfo[0]->sid); +// __block_search_fn_t searchFn = vnodeSearchKeyFunc[pTempMeter->searchAlgorithm]; + +// dTrace("QInfo:%p start to check data blocks in %d files", pQInfo, pVnodeFileInfo->numOfFiles); + + tsdb_query_handle_t *pQueryHandle = pRuntimeEnv->pQueryHandle; + while (tsdbNextDataBlock(pQueryHandle)) { + if (isQueryKilled(pQuery)) { + break; + } + + // prepare the STableDataInfo struct for each table + + SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle); +// SMeterObj * pMeterObj = getMeterObj(pSupporter->pMetersHashTable, blockInfo.sid); + +// pQInfo->pObj = pMeterObj; +// pRuntimeEnv->pMeterObj = pMeterObj; + + STableDataInfo *pTableDataInfo = NULL; +// for (int32_t i = 0; i < pSupporter->pSidSet->numOfSids; ++i) { +// if (pMeterDataInfo[i].pMeterObj == pMeterObj) { +// pTableDataInfo = &pMeterDataInfo[i]; +// break; +// } +// } + + assert(pTableDataInfo != NULL); + STableQueryInfo *pTableQueryInfo = pTableDataInfo->pTableQInfo; + + if (pTableDataInfo->pTableQInfo == NULL) { +// pTableDataInfo->pTableQInfo = createMeterQueryInfo(pQInfo, pMeterObj->sid, pQuery->skey, pQuery->ekey); + } + + restoreIntervalQueryRange(pRuntimeEnv, pTableQueryInfo); + + SDataStatis *pStatis = NULL; + SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); + + TSKEY nextKey = blockInfo.window.ekey; + if (pQuery->intervalTime == 0) { + setExecutionContext(pQInfo, pTableQueryInfo, pTableDataInfo->tableIndex, pTableDataInfo->groupIdx, + nextKey); + } else { // interval query + setIntervalQueryRange(pTableQueryInfo, pQInfo, nextKey); + int32_t ret = setAdditionalInfo(pQInfo, pTableDataInfo->tableIndex, pTableQueryInfo); + if (ret != TSDB_CODE_SUCCESS) { +// pQInfo->killed = 1; + return; + } + } + +// stableApplyFunctionsOnBlock_(pSupporter, pTableDataInfo, &blockInfo, pStatis, pDataBlock, searchFn); + } +} + +static bool multimeterMultioutputHelper(SQInfo *pQInfo, bool *dataInDisk, bool *dataInCache, int32_t index, + int32_t start) { +// SMeterSidExtInfo **pMeterSidExtInfo = pQInfo->pMeterSidExtInfo; + SQueryRuntimeEnv * pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = &pRuntimeEnv->pQuery; +#if 0 + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); + + SMeterObj *pMeterObj = getMeterObj(pSupporter->pMetersHashTable, pMeterSidExtInfo[index]->sid); + if (pMeterObj == NULL) { + dError("QInfo:%p do not find required meter id: %d, all meterObjs id is:", pQInfo, pMeterSidExtInfo[index]->sid); + return false; + } + + vnodeSetTagValueInParam(pSupporter->pSidSet, pRuntimeEnv, pMeterSidExtInfo[index]); + + dTrace("QInfo:%p query on (%d): vid:%d sid:%d meterId:%s, qrange:%" PRId64 "-%" PRId64, pQInfo, index - start, + pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->skey, pQuery->ekey); + + pQInfo->pObj = pMeterObj; + pQuery->lastKey = pQuery->skey; + pRuntimeEnv->pMeterObj = pMeterObj; + + vnodeUpdateQueryColumnIndex(pQuery, pRuntimeEnv->pMeterObj); + vnodeUpdateFilterColumnIndex(pQuery); + + vnodeCheckIfDataExists(pRuntimeEnv, pMeterObj, dataInDisk, dataInCache); + + // data in file or cache is not qualified for the query. abort + if (!(dataInCache || dataInDisk)) { + dTrace("QInfo:%p vid:%d sid:%d meterId:%s, qrange:%" PRId64 "-%" PRId64 ", nores, %p", pQInfo, pMeterObj->vnode, + pMeterObj->sid, pMeterObj->meterId, pQuery->skey, pQuery->ekey, pQuery); + return false; + } + + if (pRuntimeEnv->pTSBuf != NULL) { + if (pRuntimeEnv->cur.vnodeIndex == -1) { + int64_t tag = pRuntimeEnv->pCtx[0].tag.i64Key; + STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, tag); + + // failed to find data with the specified tag value + if (elem.vnode < 0) { + return false; + } + } else { + tsBufSetCursor(pRuntimeEnv->pTSBuf, &pRuntimeEnv->cur); + } + } + +#endif + + initCtxOutputBuf(pRuntimeEnv); + return true; +} + +static int64_t doCheckMetersInGroup(SQInfo *pQInfo, int32_t index, int32_t start) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery* pQuery = pRuntimeEnv->pQuery; + + bool dataInDisk = true; + bool dataInCache = true; + if (!multimeterMultioutputHelper(pQInfo, &dataInDisk, &dataInCache, index, start)) { + return 0; + } + + + SPointInterpoSupporter pointInterpSupporter = {0}; + pointInterpSupporterInit(pQuery, &pointInterpSupporter); + assert(0); + +// if (!normalizedFirstQueryRange(dataInDisk, dataInCache, pSupporter, &pointInterpSupporter, NULL)) { +// pointInterpSupporterDestroy(&pointInterpSupporter); +// return 0; +// } + + /* + * here we set the value for before and after the specified time into the + * parameter for interpolation query + */ + pointInterpSupporterSetData(pQInfo, &pointInterpSupporter); + pointInterpSupporterDestroy(&pointInterpSupporter); + + vnodeScanAllData(pRuntimeEnv); + + // first/last_row query, do not invoke the finalize for super table query + doFinalizeResult(pRuntimeEnv); + + int64_t numOfRes = getNumOfResult(pRuntimeEnv); + assert(numOfRes == 1 || numOfRes == 0); + + // accumulate the point interpolation result + if (numOfRes > 0) { + pQuery->rec.pointsRead += numOfRes; + forwardCtxOutputBuf(pRuntimeEnv, numOfRes); + } + + return numOfRes; +} + +/** + * super table query handler + * 1. super table projection query, group-by on normal columns query, ts-comp query + * 2. point interpolation query, last row query + * + * @param pQInfo + */ +static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { + SQueryRuntimeEnv * pRuntimeEnv = &pQInfo->runtimeEnv; + +#if 0 + SQuery* pQuery = pRuntimeEnv->pQuery; +// tSidSet *pSids = pSupporter->pSidSet; + + int32_t vid = getMeterObj(pSupporter->pMetersHashTable, pMeterSidExtInfo[0]->sid)->vnode; + + if (isPointInterpoQuery(pQuery)) { + resetCtxOutputBuf(pRuntimeEnv); + + assert(pQuery->limit.offset == 0 && pQuery->limit.limit != 0); + + while (pSupporter->subgroupIdx < pSids->numOfSubSet) { + int32_t start = pSids->starterPos[pSupporter->subgroupIdx]; + int32_t end = pSids->starterPos[pSupporter->subgroupIdx + 1] - 1; + + if (isFirstLastRowQuery(pQuery)) { + dTrace("QInfo:%p last_row query on vid:%d, numOfGroups:%d, current group:%d", pQInfo, vid, pSids->numOfSubSet, + pSupporter->subgroupIdx); + + TSKEY key = -1; + int32_t index = -1; + + // choose the last key for one group + pSupporter->meterIdx = start; + + for (int32_t k = start; k <= end; ++k, pSupporter->meterIdx++) { + if (isQueryKilled(pQuery)) { + setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); + return; + } + + // get the last key of meters that belongs to this group + SMeterObj *pMeterObj = getMeterObj(pSupporter->pMetersHashTable, pMeterSidExtInfo[k]->sid); + if (pMeterObj != NULL) { + if (key < pMeterObj->lastKey) { + key = pMeterObj->lastKey; + index = k; + } + } + } + + pQuery->skey = key; + pQuery->ekey = key; + pSupporter->rawSKey = key; + pSupporter->rawEKey = key; + + int64_t num = doCheckMetersInGroup(pQInfo, index, start); + assert(num >= 0); + } else { + dTrace("QInfo:%p interp query on vid:%d, numOfGroups:%d, current group:%d", pQInfo, vid, pSids->numOfSubSet, + pSupporter->subgroupIdx); + + for (int32_t k = start; k <= end; ++k) { + if (isQueryKilled(pQuery)) { + setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); + return; + } + + pQuery->skey = pSupporter->rawSKey; + pQuery->ekey = pSupporter->rawEKey; + + int64_t num = doCheckMetersInGroup(pQInfo, k, start); + if (num == 1) { + break; + } + } + } + + pSupporter->subgroupIdx++; + + // output buffer is full, return to client + if (pQuery->pointsRead >= pQuery->pointsToRead) { + break; + } + } + } else { + /* + * 1. super table projection query, 2. group-by on normal columns query, 3. ts-comp query + */ + assert(pSupporter->meterIdx >= 0); + + /* + * if the subgroup index is larger than 0, results generated by group by tbname,k is existed. + * we need to return it to client in the first place. + */ + if (pSupporter->subgroupIdx > 0) { + copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); + pQInfo->pointsRead += pQuery->pointsRead; + + if (pQuery->pointsRead > 0) { + return; + } + } + + if (pSupporter->meterIdx >= pSids->numOfSids) { + return; + } + + resetCtxOutputBuf(pRuntimeEnv); + resetTimeWindowInfo(pRuntimeEnv, &pRuntimeEnv->windowResInfo); + + while (pSupporter->meterIdx < pSupporter->numOfMeters) { + int32_t k = pSupporter->meterIdx; + + if (isQueryKilled(pQuery)) { + setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); + return; + } + + TSKEY skey = pQInfo->pTableQuerySupporter->pMeterSidExtInfo[k]->key; + if (skey > 0) { + pQuery->skey = skey; + } + + bool dataInDisk = true; + bool dataInCache = true; + if (!multimeterMultioutputHelper(pQInfo, &dataInDisk, &dataInCache, k, 0)) { + pQuery->skey = pSupporter->rawSKey; + pQuery->ekey = pSupporter->rawEKey; + + pSupporter->meterIdx++; + continue; + } + +#if DEFAULT_IO_ENGINE == IO_ENGINE_MMAP + for (int32_t i = 0; i < pRuntimeEnv->numOfFiles; ++i) { + resetMMapWindow(&pRuntimeEnv->pVnodeFiles[i]); + } +#endif + + SPointInterpoSupporter pointInterpSupporter = {0}; + assert(0); +// if (normalizedFirstQueryRange(dataInDisk, dataInCache, pSupporter, &pointInterpSupporter, NULL) == false) { +// pQuery->skey = pSupporter->rawSKey; +// pQuery->ekey = pSupporter->rawEKey; +// +// pSupporter->meterIdx++; +// continue; +// } + + // TODO handle the limit problem + if (pQuery->numOfFilterCols == 0 && pQuery->limit.offset > 0) { + forwardQueryStartPosition(pRuntimeEnv); + + if (Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED)) { + pQuery->skey = pSupporter->rawSKey; + pQuery->ekey = pSupporter->rawEKey; + + pSupporter->meterIdx++; + continue; + } + } + + vnodeScanAllData(pRuntimeEnv); + + pQuery->pointsRead = getNumOfResult(pRuntimeEnv); + doSkipResults(pRuntimeEnv); + + // the limitation of output result is reached, set the query completed + if (doRevisedResultsByLimit(pQInfo)) { + pSupporter->meterIdx = pSupporter->pSidSet->numOfSids; + break; + } + + // enable execution for next table, when handling the projection query + enableExecutionForNextTable(pRuntimeEnv); + + if (Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED)) { + /* + * query range is identical in terms of all meters involved in query, + * so we need to restore them at the *beginning* of query on each meter, + * not the consecutive query on meter on which is aborted due to buffer limitation + * to ensure that, we can reset the query range once query on a meter is completed. + */ + pQuery->skey = pSupporter->rawSKey; + pQuery->ekey = pSupporter->rawEKey; + pSupporter->meterIdx++; + + pQInfo->pTableQuerySupporter->pMeterSidExtInfo[k]->key = pQuery->lastKey; + + // if the buffer is full or group by each table, we need to jump out of the loop + if (Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL) || + isGroupbyEachTable(pQuery->pGroupbyExpr, pSupporter->pSidSet)) { + break; + } + + } else { // forward query range + pQuery->skey = pQuery->lastKey; + + // all data in the result buffer are skipped due to the offset, continue to retrieve data from current meter + if (pQuery->pointsRead == 0) { + assert(!Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL)); + continue; + } else { + pQInfo->pTableQuerySupporter->pMeterSidExtInfo[k]->key = pQuery->lastKey; + // buffer is full, wait for the next round to retrieve data from current meter + assert(Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL)); + break; + } + } + } + } + + /* + * 1. super table projection query, group-by on normal columns query, ts-comp query + * 2. point interpolation query, last row query + * + * group-by on normal columns query and last_row query do NOT invoke the finalizer here, + * since the finalize stage will be done at the client side. + * + * projection query, point interpolation query do not need the finalizer. + * + * Only the ts-comp query requires the finalizer function to be executed here. + */ + if (isTSCompQuery(pQuery)) { + doFinalizeResult(pRuntimeEnv); + } + + if (pRuntimeEnv->pTSBuf != NULL) { + pRuntimeEnv->cur = pRuntimeEnv->pTSBuf->cur; + } + + // todo refactor + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + + for (int32_t i = 0; i < pWindowResInfo->size; ++i) { + SWindowStatus *pStatus = &pWindowResInfo->pResult[i].status; + pStatus->closed = true; // enable return all results for group by normal columns + + SWindowResult *pResult = &pWindowResInfo->pResult[i]; + for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes); + } + } + + pQInfo->pTableQuerySupporter->subgroupIdx = 0; + pQuery->pointsRead = 0; + copyFromWindowResToSData(pQInfo, pWindowResInfo->pResult); + } + + pQInfo->pointsRead += pQuery->pointsRead; + pQuery->pointsOffset = pQuery->pointsToRead; + + dTrace( + "QInfo %p vid:%d, numOfMeters:%d, index:%d, numOfGroups:%d, %d points returned, totalRead:%d totalReturn:%d," + "next skey:%" PRId64 ", offset:%" PRId64, + pQInfo, vid, pSids->numOfSids, pSupporter->meterIdx, pSids->numOfSubSet, pQuery->pointsRead, pQInfo->pointsRead, + pQInfo->pointsReturned, pQuery->skey, pQuery->limit.offset); +#endif +} + +static void doOrderedScan(SQInfo *pQInfo) { + SQuery *pQuery = &pQInfo->runtimeEnv.pQuery; +#if 0 +// if (pQInfo->runtimeEnv. == NULL) { +// pSupporter->pMeterDataInfo = calloc(pSupporter->pSidSet->numOfSids, sizeof(STableDataInfo)); +// } + + SMeterSidExtInfo **pMeterSidExtInfo = pSupporter->pMeterSidExtInfo; + + tSidSet* pSidset = pSupporter->pSidSet; + int32_t groupId = 0; + + for (int32_t i = 0; i < pSidset->numOfSids; ++i) { // load all meter meta info + SMeterObj *pMeterObj = getMeterObj(pSupporter->pMetersHashTable, pMeterSidExtInfo[i]->sid); + if (pMeterObj == NULL) { + dError("QInfo:%p failed to find required sid:%d", pQInfo, pMeterSidExtInfo[i]->sid); + continue; + } + + if (i >= pSidset->starterPos[groupId + 1]) { + groupId += 1; + } + + STableDataInfo *pOneMeterDataInfo = &pSupporter->pMeterDataInfo[i]; + assert(pOneMeterDataInfo->pMeterObj == NULL); + + setMeterDataInfo(pOneMeterDataInfo, pMeterObj, i, groupId); + pOneMeterDataInfo->pTableQInfo = createMeterQueryInfo(pSupporter, pMeterObj->sid, pQuery->skey, pQuery->ekey); + } + + queryOnDataBlocks(pQInfo, pSupporter->pMeterDataInfo); + if (pQInfo->code != TSDB_CODE_SUCCESS) { + return; + } +#endif +} + +static void setupMeterQueryInfoForSupplementQuery(SQInfo *pQInfo) { + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + int32_t num = taosHashGetSize(pQInfo->pTableList); + for (int32_t i = 0; i < num; ++i) { +// STableQueryInfo *pTableQueryInfo = pSupporter->pMeterDataInfo[i].pTableQInfo; +// changeMeterQueryInfoForSuppleQuery(pQuery, pTableQueryInfo, pSupporter->rawSKey, pSupporter->rawEKey); + } +} + +static void doMultiMeterSupplementaryScan(SQInfo *pQInfo) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery *pQuery = pRuntimeEnv->pQuery; + + if (!needSupplementaryScan(pQuery)) { + dTrace("QInfo:%p no need to do supplementary scan, query completed", pQInfo); + return; + } + + SET_SUPPLEMENT_SCAN_FLAG(pRuntimeEnv); +// disableFunctForSuppleScan(pSupporter, pQuery->order.order); + + if (pRuntimeEnv->pTSBuf != NULL) { + pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1u; + } + +#if 0 + SWAP(pSupporter->rawSKey, pSupporter->rawEKey, TSKEY); + setupMeterQueryInfoForSupplementQuery(pSupporter); + + int64_t st = taosGetTimestampMs(); + + doOrderedScan(pQInfo); + + int64_t et = taosGetTimestampMs(); + dTrace("QInfo:%p supplementary scan completed, elapsed time: %lldms", pQInfo, et - st); + + /* + * restore the env + * the meter query info is not reset to the original state + */ + SWAP(pSupporter->rawSKey, pSupporter->rawEKey, TSKEY); + enableFunctForMasterScan(pRuntimeEnv, pQuery->order.order); + + if (pRuntimeEnv->pTSBuf != NULL) { + pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1; + } +#endif + SET_MASTER_SCAN_FLAG(pRuntimeEnv); +} + +static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { + SQueryRuntimeEnv * pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + + if (pQInfo->subgroupIdx > 0) { + /* + * if the subgroupIdx > 0, the query process must be completed yet, we only need to + * copy the data into output buffer + */ + if (pQuery->intervalTime > 0) { + copyResToQueryResultBuf(pQInfo, pQuery); + +#ifdef _DEBUG_VIEW + displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->len); +#endif + } else { + copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); + } + + pQInfo->rec.pointsRead += pQuery->rec.pointsRead; + + if (pQuery->rec.pointsRead == 0) { +// vnodePrintQueryStatistics(pSupporter); + } + + dTrace("QInfo:%p points returned:%d, totalRead:%d totalReturn:%d", pQInfo, pQuery->rec.pointsRead, + pQInfo->rec.pointsRead, pQInfo->pointsReturned); + return; + } +#if 0 + pSupporter->pMeterDataInfo = (STableDataInfo *)calloc(1, sizeof(STableDataInfo) * pSupporter->numOfMeters); + if (pSupporter->pMeterDataInfo == NULL) { + dError("QInfo:%p failed to allocate memory, %s", pQInfo, strerror(errno)); + pQInfo->code = -TSDB_CODE_SERV_OUT_OF_MEMORY; + return; + } + + dTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", order:%d, group:%d", pQInfo, pSupporter->rawSKey, + pSupporter->rawEKey, pQuery->order.order, pSupporter->pSidSet->numOfSubSet); + + dTrace("QInfo:%p main query scan start", pQInfo); + int64_t st = taosGetTimestampMs(); + doOrderedScan(pQInfo); + + int64_t et = taosGetTimestampMs(); + dTrace("QInfo:%p main scan completed, elapsed time: %lldms, supplementary scan start, order:%d", pQInfo, et - st, + pQuery->order.order ^ 1u); + + if (pQuery->intervalTime > 0) { + for (int32_t i = 0; i < pSupporter->numOfMeters; ++i) { + STableQueryInfo *pTableQueryInfo = pSupporter->pMeterDataInfo[i].pTableQInfo; + closeAllTimeWindow(&pTableQueryInfo->windowResInfo); + } + } else { // close results for group result + closeAllTimeWindow(&pRuntimeEnv->windowResInfo); + } + + doMultiMeterSupplementaryScan(pQInfo); + + if (isQueryKilled(pQuery)) { + dTrace("QInfo:%p query killed, abort", pQInfo); + return; + } + + if (pQuery->intervalTime > 0 || isSumAvgRateQuery(pQuery)) { + assert(pSupporter->subgroupIdx == 0 && pSupporter->numOfGroupResultPages == 0); + + if (mergeMetersResultToOneGroups(pSupporter) == TSDB_CODE_SUCCESS) { + copyResToQueryResultBuf(pSupporter, pQuery); + +#ifdef _DEBUG_VIEW + displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->len); +#endif + } + } else { // not a interval query + copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); + } + + // handle the limitation of output buffer + pQInfo->pointsRead += pQuery->pointsRead; + dTrace("QInfo:%p points returned:%d, totalRead:%d totalReturn:%d", pQInfo, pQuery->pointsRead, pQInfo->pointsRead, + pQInfo->pointsReturned); +#endif +} + +/* + * in each query, this function will be called only once, no retry for further result. + * + * select count(*)/top(field,k)/avg(field name) from table_name [where ts>now-1a]; + * select count(*) from table_name group by status_column; + */ +static void vnodeSingleTableFixedOutputProcessor(SQInfo *pQInfo) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + + vnodeScanAllData(pRuntimeEnv); + doFinalizeResult(pRuntimeEnv); + + if (isQueryKilled(pQuery)) { + return; + } + + // since the numOfOutputElems must be identical for all sql functions that are allowed to be executed simutanelously. + pQuery->rec.pointsRead = getNumOfResult(pRuntimeEnv); + // assert(pQuery->pointsRead <= pQuery->pointsToRead && + // Q_STATUS_EQUAL(pQuery->over, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)); + + // must be top/bottom query if offset > 0 + if (pQuery->limit.offset > 0) { + assert(isTopBottomQuery(pQuery)); + } + + doSkipResults(pRuntimeEnv); + doRevisedResultsByLimit(pQInfo); + + pQInfo->rec.pointsRead = pQuery->rec.pointsRead; +} + +static void vnodeSingleTableMultiOutputProcessor(SQInfo *pQInfo) { +#if 0 + SQuery * pQuery = &pQInfo->query; + SMeterObj *pMeterObj = pQInfo->pObj; + + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->pTableQuerySupporter->runtimeEnv; + + // for ts_comp query, re-initialized is not allowed + if (!isTSCompQuery(pQuery)) { + resetCtxOutputBuf(pRuntimeEnv); + } + + while (1) { + vnodeScanAllData(pRuntimeEnv); + doFinalizeResult(pRuntimeEnv); + + if (isQueryKilled(pQuery)) { + return; + } + + pQuery->pointsRead = getNumOfResult(pRuntimeEnv); + if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols > 0 && pQuery->pointsRead > 0) { + doSkipResults(pRuntimeEnv); + } + + /* + * 1. if pQuery->pointsRead == 0, pQuery->limit.offset >= 0, still need to check data + * 2. if pQuery->pointsRead > 0, pQuery->limit.offset must be 0 + */ + if (pQuery->pointsRead > 0 || Q_STATUS_EQUAL(pQuery->over, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)) { + break; + } + + TSKEY nextTimestamp = loadRequiredBlockIntoMem(pRuntimeEnv, &pRuntimeEnv->nextPos); + assert(nextTimestamp > 0 || ((nextTimestamp < 0) && Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK))); + + dTrace("QInfo:%p vid:%d sid:%d id:%s, skip current result, offset:%" PRId64 ", next qrange:%" PRId64 "-%" PRId64, + pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->limit.offset, pQuery->lastKey, + pQuery->ekey); + + resetCtxOutputBuf(pRuntimeEnv); + } + + doRevisedResultsByLimit(pQInfo); + pQInfo->pointsRead += pQuery->pointsRead; + + if (Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL)) { + TSKEY nextTimestamp = loadRequiredBlockIntoMem(pRuntimeEnv, &pRuntimeEnv->nextPos); + assert(nextTimestamp > 0 || ((nextTimestamp < 0) && Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK))); + + dTrace("QInfo:%p vid:%d sid:%d id:%s, query abort due to buffer limitation, next qrange:%" PRId64 "-%" PRId64, + pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->lastKey, pQuery->ekey); + } + + dTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned, totalRead:%d totalReturn:%d", pQInfo, pMeterObj->vnode, + pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, pQInfo->pointsRead, pQInfo->pointsReturned); + + pQuery->pointsOffset = pQuery->pointsToRead; // restore the available buffer + if (!isTSCompQuery(pQuery)) { + assert(pQuery->pointsRead <= pQuery->pointsToRead); + } + +#endif +} + +static void vnodeSingleMeterIntervalMainLooper(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + while (1) { + initCtxOutputBuf(pRuntimeEnv); + vnodeScanAllData(pRuntimeEnv); + + if (isQueryKilled(pQuery)) { + return; + } + + assert(!Q_STATUS_EQUAL(pQuery->status, QUERY_NOT_COMPLETED)); + doFinalizeResult(pRuntimeEnv); + + // here we can ignore the records in case of no interpolation + // todo handle offset, in case of top/bottom interval query + if ((pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) && pQuery->limit.offset > 0 && + pQuery->interpoType == TSDB_INTERPO_NONE) { + // maxOutput <= 0, means current query does not generate any results + int32_t numOfClosed = numOfClosedTimeWindow(&pRuntimeEnv->windowResInfo); + + int32_t c = MIN(numOfClosed, pQuery->limit.offset); + clearFirstNTimeWindow(pRuntimeEnv, c); + pQuery->limit.offset -= c; + } + + if (Q_STATUS_EQUAL(pQuery->status, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED)) { + break; + } + + // load the data block for the next retrieve +// loadRequiredBlockIntoMem(pRuntimeEnv, &pRuntimeEnv->nextPos); + if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { + break; + } + } +} + +/* handle time interval query on single table */ +static void vnodeSingleTableIntervalProcessor(SQInfo *pQInfo) { +// STable *pMeterObj = pQInfo->pObj; + + SQueryRuntimeEnv * pRuntimeEnv = &(pQInfo->runtimeEnv); + SQuery* pQuery = pRuntimeEnv->pQuery; + + int32_t numOfInterpo = 0; + + while (1) { + resetCtxOutputBuf(pRuntimeEnv); + vnodeSingleMeterIntervalMainLooper(pRuntimeEnv); + + if (pQuery->intervalTime > 0) { + pQInfo->subgroupIdx = 0; // always start from 0 + pQuery->rec.pointsRead = 0; + copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); + + clearFirstNTimeWindow(pRuntimeEnv, pQInfo->subgroupIdx); + } + + // the offset is handled at prepare stage if no interpolation involved + if (pQuery->interpoType == TSDB_INTERPO_NONE) { + doRevisedResultsByLimit(pQInfo); + break; + } else { + taosInterpoSetStartInfo(&pRuntimeEnv->interpoInfo, pQuery->rec.pointsRead, pQuery->interpoType); + SData **pInterpoBuf = pRuntimeEnv->pInterpoBuf; + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + memcpy(pInterpoBuf[i]->data, pQuery->sdata[i]->data, pQuery->rec.pointsRead * pQuery->pSelectExpr[i].resBytes); + } + + numOfInterpo = 0; + pQuery->rec.pointsRead = vnodeQueryResultInterpolate(pQInfo, (tFilePage **)pQuery->sdata, (tFilePage **)pInterpoBuf, + pQuery->rec.pointsRead, &numOfInterpo); + + dTrace("QInfo: %p interpo completed, final:%d", pQInfo, pQuery->rec.pointsRead); + if (pQuery->rec.pointsRead > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)) { + doRevisedResultsByLimit(pQInfo); + break; + } + + // no result generated yet, continue retrieve data + pQuery->rec.pointsRead = 0; + } + } + + // all data scanned, the group by normal column can return + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // todo refactor with merge interval time result + pQInfo->subgroupIdx = 0; + pQuery->rec.pointsRead = 0; + copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); + clearFirstNTimeWindow(pRuntimeEnv, pQInfo->subgroupIdx); + } + + pQInfo->rec.pointsRead += pQuery->rec.pointsRead; + pQInfo->pointsInterpo += numOfInterpo; + +// dTrace("%p vid:%d sid:%d id:%s, %d points returned %d points interpo, totalRead:%d totalInterpo:%d totalReturn:%d", +// pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, numOfInterpo, +// pQInfo->pointsRead - pQInfo->pointsInterpo, pQInfo->pointsInterpo, pQInfo->pointsReturned); +} + +void qTableQuery(void* pReadMsg) { +// SQInfo *pQInfo = (SQInfo *)pReadMsg->ahandle; + +#if 0 + if (pQInfo == NULL) { + dTrace("%p freed abort query", pQInfo); + return; + } + +// if (pQInfo->killed) { +// dTrace("QInfo:%p it is already killed, abort", pQInfo); +// vnodeDecRefCount(pQInfo); +// +// return; +// } + +// assert(pQInfo->refCount >= 1); + + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = &pRuntimeEnv->pQuery; + +// assert(pRuntimeEnv->pMeterObj == pMeterObj); + +// dTrace("vid:%d sid:%d id:%s, query thread is created, numOfQueries:%d, QInfo:%p", pMeterObj->vnode, pMeterObj->sid, +// pMeterObj->meterId, pMeterObj->numOfQueries, pQInfo); + + if (vnodeHasRemainResults(pQInfo)) { + /* + * There are remain results that are not returned due to result interpolation + * So, we do keep in this procedure instead of launching retrieve procedure for next results. + */ + int32_t numOfInterpo = 0; + + int32_t remain = taosNumOfRemainPoints(&pRuntimeEnv->interpoInfo); + pQuery->rec.pointsRead = vnodeQueryResultInterpolate(pQInfo, (tFilePage **)pQuery->sdata, + (tFilePage **)pRuntimeEnv->pInterpoBuf, remain, &numOfInterpo); + + doRevisedResultsByLimit(pQInfo); + + pQInfo->pointsInterpo += numOfInterpo; + pQInfo->rec.pointsRead += pQuery->rec.pointsRead; + +// dTrace( +// "QInfo:%p vid:%d sid:%d id:%s, %d points returned %d points interpo, totalRead:%d totalInterpo:%d " +// "totalReturn:%d", +// pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, numOfInterpo, +// pQInfo->pointsRead, pQInfo->pointsInterpo, pQInfo->pointsReturned); + + sem_post(&pQInfo->dataReady); +// vnodeDecRefCount(pQInfo); + + return; + } + + // here we have scan all qualified data in both data file and cache + if (Q_STATUS_EQUAL(pQuery->status, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED)) { + // continue to get push data from the group result + if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || + (pQuery->intervalTime > 0 && pQInfo->pointsReturned < pQuery->limit.limit)) { + // todo limit the output for interval query? + pQuery->rec.pointsRead = 0; + pQInfo->subgroupIdx = 0; // always start from 0 + + if (pRuntimeEnv->windowResInfo.size > 0) { + copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); + pQInfo->rec.pointsRead += pQuery->rec.pointsRead; + + clearFirstNTimeWindow(pRuntimeEnv, pQInfo->subgroupIdx); + + if (pQuery->rec.pointsRead > 0) { +// dTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned %d from group results, totalRead:%d totalReturn:%d", +// pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, pQInfo->pointsRead, +// pQInfo->pointsInterpo, pQInfo->pointsReturned); + + sem_post(&pQInfo->dataReady); +// vnodeDecRefCount(pQInfo); + + return; + } + } + } + + assert(0); +// pQInfo->over = 1; +// dTrace("QInfo:%p vid:%d sid:%d id:%s, query over, %d points are returned", pQInfo, pMeterObj->vnode, pMeterObj->sid, +// pMeterObj->meterId, pQInfo->pointsRead); + +// vnodePrintQueryStatistics(pSupporter); + sem_post(&pQInfo->dataReady); + +// vnodeDecRefCount(pQInfo); + return; + } + + /* number of points returned during this query */ + pQuery->rec.pointsRead = 0; + + int64_t st = taosGetTimestampUs(); + + // group by normal column, sliding window query, interval query are handled by interval query processor + if (pQuery->intervalTime != 0 || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // interval (down sampling operation) +// assert(pQuery->checkBufferInLoop == 0 && pQuery->pointsOffset == pQuery->pointsToRead); + vnodeSingleTableIntervalProcessor(pQInfo); + } else { + if (isFixedOutputQuery(pQuery)) { + assert(pQuery->checkBufferInLoop == 0); + vnodeSingleTableFixedOutputProcessor(pQInfo); + } else { // diff/add/multiply/subtract/division + assert(pQuery->checkBufferInLoop == 1); + vnodeSingleTableMultiOutputProcessor(pQInfo); + } + } + + // record the total elapsed time + pQInfo->elapsedTime += (taosGetTimestampUs() - st); + + /* check if query is killed or not */ + if (isQueryKilled(pQuery)) { + dTrace("QInfo:%p query is killed", pQInfo); +// pQInfo->over = 1; + } else { +// dTrace("QInfo:%p vid:%d sid:%d id:%s, meter query thread completed, %d points are returned", pQInfo, +// pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead); + } + + sem_post(&pQInfo->dataReady); +// vnodeDecRefCount(pQInfo); +#endif +} + +void qSuperTableQuery(void *pReadMsg) { +// SQInfo *pQInfo = (SQInfo *)pMsg->ahandle; +// +// if (pQInfo == NULL) { +// return; +// } + +// if (pQInfo->killed) { +// vnodeDecRefCount(pQInfo); +// dTrace("QInfo:%p it is already killed, abort", pQInfo); +// return; +// } + +// assert(pQInfo->refCount >= 1); +#if 0 + SQuery *pQuery = &pQInfo->runtimeEnv.pQuery; + pQuery->rec.pointsRead = 0; + + int64_t st = taosGetTimestampUs(); + if (pQuery->intervalTime > 0 || + (isFixedOutputQuery(pQuery) && (!isPointInterpoQuery(pQuery)) && !isGroupbyNormalCol(pQuery->pGroupbyExpr))) { + assert(pQuery->checkBufferInLoop == 0); + vnodeMultiMeterQueryProcessor(pQInfo); + } else { + assert((pQuery->checkBufferInLoop == 1 && pQuery->intervalTime == 0) || isPointInterpoQuery(pQuery) || + isGroupbyNormalCol(pQuery->pGroupbyExpr)); + + vnodeSTableSeqProcessor(pQInfo); + } + + /* record the total elapsed time */ + pQInfo->elapsedTime += (taosGetTimestampUs() - st); + pQuery->status = isQueryKilled(pQuery) ? 1 : 0; + +// taosInterpoSetStartInfo(&pQInfo->runtimeEnv.interpoInfo, pQuery->pointsRead, +// pQInfo->query.interpoType); + + if (pQuery->rec.pointsRead == 0) { +// pQInfo->over = 1; +// dTrace("QInfo:%p over, %d meters queried, %d points are returned", pQInfo, pSupporter->numOfMeters, +// pQInfo->pointsRead); +// vnodePrintQueryStatistics(pSupporter); + } + + sem_post(&pQInfo->dataReady); +// vnodeDecRefCount(pQInfo); +#endif +} diff --git a/src/query/src/queryUtil.c b/src/query/src/queryUtil.c new file mode 100644 index 0000000000000000000000000000000000000000..c970363c01fc148a58da8dd78cf0a43c255297cc --- /dev/null +++ b/src/query/src/queryUtil.c @@ -0,0 +1,269 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" + +#include "hash.h" +#include "taosmsg.h" +#include "qextbuffer.h" +#include "ttime.h" + +#include "qinterpolation.h" +//#include "tscJoinProcess.h" +#include "ttime.h" + +#include "queryExecutor.h" + +int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRuntimeEnv, int32_t size, + int32_t threshold, int16_t type) { + pWindowResInfo->capacity = size; + pWindowResInfo->threshold = threshold; + + pWindowResInfo->type = type; + + _hash_fn_t fn = taosGetDefaultHashFunction(type); + pWindowResInfo->hashList = taosHashInit(threshold, fn, false); + + pWindowResInfo->curIndex = -1; + pWindowResInfo->size = 0; + + // use the pointer arraylist + pWindowResInfo->pResult = calloc(threshold, sizeof(SWindowResult)); + for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { + SPosInfo posInfo = {-1, -1}; + createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, &posInfo); + } + + return TSDB_CODE_SUCCESS; +} + +void destroyTimeWindowRes(SWindowResult *pWindowRes, int32_t nOutputCols) { + if (pWindowRes == NULL) { + return; + } + + for (int32_t i = 0; i < nOutputCols; ++i) { + free(pWindowRes->resultInfo[i].interResultBuf); + } + + free(pWindowRes->resultInfo); +} + +void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo, int32_t numOfCols) { + if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0) { + assert(pWindowResInfo->hashList == NULL && pWindowResInfo->pResult == NULL); + return; + } + + for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { + SWindowResult *pResult = &pWindowResInfo->pResult[i]; + destroyTimeWindowRes(pResult, numOfCols); + } + + taosHashCleanup(pWindowResInfo->hashList); + tfree(pWindowResInfo->pResult); +} + +void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo) { + if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0) { + return; + } + + for (int32_t i = 0; i < pWindowResInfo->size; ++i) { + SWindowResult *pWindowRes = &pWindowResInfo->pResult[i]; + clearTimeWindowResBuf(pRuntimeEnv, pWindowRes); + } + + pWindowResInfo->curIndex = -1; + taosHashCleanup(pWindowResInfo->hashList); + pWindowResInfo->size = 0; + + _hash_fn_t fn = taosGetDefaultHashFunction(pWindowResInfo->type); + pWindowResInfo->hashList = taosHashInit(pWindowResInfo->capacity, fn, false); + + pWindowResInfo->startTime = 0; + pWindowResInfo->prevSKey = 0; +} + +void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0 || pWindowResInfo->size == 0 || num == 0) { + return; + } + + int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo); + assert(num >= 0 && num <= numOfClosed); + + for (int32_t i = 0; i < num; ++i) { + SWindowResult *pResult = &pWindowResInfo->pResult[i]; + if (pResult->status.closed) { // remove the window slot from hash table + taosHashRemove(pWindowResInfo->hashList, (const char *)&pResult->window.skey, TSDB_KEYSIZE); + } else { + break; + } + } + + int32_t remain = pWindowResInfo->size - num; + + // clear all the closed windows from the window list + for (int32_t k = 0; k < remain; ++k) { + copyTimeWindowResBuf(pRuntimeEnv, &pWindowResInfo->pResult[k], &pWindowResInfo->pResult[num + k]); + } + + // move the unclosed window in the front of the window list + for (int32_t k = remain; k < pWindowResInfo->size; ++k) { + SWindowResult *pWindowRes = &pWindowResInfo->pResult[k]; + clearTimeWindowResBuf(pRuntimeEnv, pWindowRes); + } + + pWindowResInfo->size = remain; + + for (int32_t k = 0; k < pWindowResInfo->size; ++k) { + SWindowResult *pResult = &pWindowResInfo->pResult[k]; + int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)&pResult->window.skey, TSDB_KEYSIZE); + int32_t v = (*p - num); + assert(v >= 0 && v <= pWindowResInfo->size); + + taosHashPut(pWindowResInfo->hashList, (const char *)&pResult->window.skey, TSDB_KEYSIZE, (char *)&v, + sizeof(int32_t)); + } + + pWindowResInfo->curIndex = -1; +} + +void clearClosedTimeWindow(SQueryRuntimeEnv *pRuntimeEnv) { + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0 || pWindowResInfo->size == 0) { + return; + } + + int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo); + clearFirstNTimeWindow(pRuntimeEnv, numOfClosed); +} + +int32_t numOfClosedTimeWindow(SWindowResInfo *pWindowResInfo) { + int32_t i = 0; + while (i < pWindowResInfo->size && pWindowResInfo->pResult[i].status.closed) { + ++i; + } + + return i; +} + +void closeAllTimeWindow(SWindowResInfo *pWindowResInfo) { + assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size); + + for (int32_t i = 0; i < pWindowResInfo->size; ++i) { + if (pWindowResInfo->pResult[i].status.closed) { + continue; + } + + pWindowResInfo->pResult[i].status.closed = true; + } +} + +/* + * remove the results that are not the FIRST time window that spreads beyond the + * the last qualified time stamp in case of sliding query, which the sliding time is not equalled to the interval time + */ +void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_t order) { + assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size); + + int32_t i = 0; + while (i < pWindowResInfo->size && + ((pWindowResInfo->pResult[i].window.ekey < lastKey && order == QUERY_ASC_FORWARD_STEP) || + (pWindowResInfo->pResult[i].window.skey > lastKey && order == QUERY_DESC_FORWARD_STEP))) { + ++i; + } + + // assert(i < pWindowResInfo->size); + if (i < pWindowResInfo->size) { + pWindowResInfo->size = (i + 1); + } +} + +SWindowResult *getWindowResult(SWindowResInfo *pWindowResInfo, int32_t slot) { + assert(pWindowResInfo != NULL && slot >= 0 && slot < pWindowResInfo->size); + return &pWindowResInfo->pResult[slot]; +} + +bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot) { + return (getWindowResult(pWindowResInfo, slot)->status.closed == true); +} + +int32_t curTimeWindow(SWindowResInfo *pWindowResInfo) { + assert(pWindowResInfo->curIndex >= 0 && pWindowResInfo->curIndex < pWindowResInfo->size); + return pWindowResInfo->curIndex; +} + +void closeTimeWindow(SWindowResInfo *pWindowResInfo, int32_t slot) { + getWindowResult(pWindowResInfo, slot)->status.closed = true; +} + +void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindowRes) { + if (pWindowRes == NULL) { + return; + } + + for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutputCols; ++i) { + SResultInfo *pResultInfo = &pWindowRes->resultInfo[i]; + + char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes); + size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].resBytes; + memset(s, 0, size); + + resetResultInfo(pResultInfo); + } + + pWindowRes->numOfRows = 0; + // pWindowRes->nAlloc = 0; + pWindowRes->pos = (SPosInfo){-1, -1}; + pWindowRes->status.closed = false; + pWindowRes->window = (STimeWindow){0, 0}; +} + +/** + * The source window result pos attribution of the source window result does not assign to the destination, + * since the attribute of "Pos" is bound to each window result when the window result is created in the + * disk-based result buffer. + */ +void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, const SWindowResult *src) { + dst->numOfRows = src->numOfRows; + // dst->nAlloc = src->nAlloc; + dst->window = src->window; + dst->status = src->status; + + int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutputCols; + + for (int32_t i = 0; i < nOutputCols; ++i) { + SResultInfo *pDst = &dst->resultInfo[i]; + SResultInfo *pSrc = &src->resultInfo[i]; + + char *buf = pDst->interResultBuf; + memcpy(pDst, pSrc, sizeof(SResultInfo)); + pDst->interResultBuf = buf; // restore the allocated buffer + + // copy the result info struct + memcpy(pDst->interResultBuf, pSrc->interResultBuf, pDst->bufLen); + + // copy the output buffer data from src to dst, the position info keep unchanged + char * dstBuf = getPosInResultPage(pRuntimeEnv, i, dst); + char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SWindowResult *)src); + size_t s = pRuntimeEnv->pQuery->pSelectExpr[i].resBytes; + + memcpy(dstBuf, srcBuf, s); + } +} + diff --git a/src/util/inc/tcache.h b/src/util/inc/tcache.h index 6f6ef17d8d34cc851111ec760ad4e19ff2788b87..36635fff30f32f3ad3a993e2e0c5139f35eec1a6 100644 --- a/src/util/inc/tcache.h +++ b/src/util/inc/tcache.h @@ -98,15 +98,15 @@ SCacheObj *taosCacheInit(void *tmrCtrl, int64_t refreshTimeInSeconds); * @param keepTime survival time in second * @return cached element */ -void *taosCachePut(void *handle, char *key, char *pData, int dataSize, int keepTimeInSeconds); +void *taosCachePut(SCacheObj *pCacheObj, char *key, void *pData, size_t dataSize, int keepTimeInSeconds); /** * get data from cache - * @param handle cache object + * @param pCacheObj cache object * @param key key * @return cached data or NULL */ -void *taosCacheAcquireByName(void *handle, char *key); +void *taosCacheAcquireByName(SCacheObj *pCacheObj, const char *key); /** * Add one reference count for the exist data, and assign this data for a new owner. @@ -118,7 +118,7 @@ void *taosCacheAcquireByName(void *handle, char *key); * @param data * @return */ -void *taosCacheAcquireByData(void *handle, void *data); +void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data); /** * transfer the ownership of data in cache to another object without increasing reference count. @@ -126,7 +126,7 @@ void *taosCacheAcquireByData(void *handle, void *data); * @param data * @return */ -void *taosCacheTransfer(void *handle, void **data); +void *taosCacheTransfer(SCacheObj *pCacheObj, void **data); /** * remove data in cache, the data will not be removed immediately. @@ -136,7 +136,7 @@ void *taosCacheTransfer(void *handle, void **data); * @param _remove force model, reduce the ref count and move the data into * pTrash */ -void taosCacheRelease(void *handle, void **data, bool _remove); +void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove); /** * move all data node into trash, clear node in trash can if it is not referenced by any clients diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index ff741d1fd95d1daea2dc1416dc8b38f2d08be180..c8c3879f0a34c5236a7b691bbc08ed05a08502cd 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -22,43 +22,43 @@ #include "ttimer.h" #include "tutil.h" -static FORCE_INLINE void __cache_wr_lock(SCacheObj *pObj) { +static FORCE_INLINE void __cache_wr_lock(SCacheObj *pCacheObj) { #if defined(LINUX) - pthread_rwlock_wrlock(&pObj->lock); + pthread_rwlock_wrlock(&pCacheObj->lock); #else - pthread_mutex_lock(&pObj->lock); + pthread_mutex_lock(&pCacheObj->lock); #endif } -static FORCE_INLINE void __cache_rd_lock(SCacheObj *pObj) { +static FORCE_INLINE void __cache_rd_lock(SCacheObj *pCacheObj) { #if defined(LINUX) - pthread_rwlock_rdlock(&pObj->lock); + pthread_rwlock_rdlock(&pCacheObj->lock); #else - pthread_mutex_lock(&pObj->lock); + pthread_mutex_lock(&pCacheObj->lock); #endif } -static FORCE_INLINE void __cache_unlock(SCacheObj *pObj) { +static FORCE_INLINE void __cache_unlock(SCacheObj *pCacheObj) { #if defined(LINUX) - pthread_rwlock_unlock(&pObj->lock); + pthread_rwlock_unlock(&pCacheObj->lock); #else - pthread_mutex_unlock(&pObj->lock); + pthread_mutex_unlock(&pCacheObj->lock); #endif } -static FORCE_INLINE int32_t __cache_lock_init(SCacheObj *pObj) { +static FORCE_INLINE int32_t __cache_lock_init(SCacheObj *pCacheObj) { #if defined(LINUX) - return pthread_rwlock_init(&pObj->lock, NULL); + return pthread_rwlock_init(&pCacheObj->lock, NULL); #else - return pthread_mutex_init(&pObj->lock, NULL); + return pthread_mutex_init(&pCacheObj->lock, NULL); #endif } -static FORCE_INLINE void __cache_lock_destroy(SCacheObj *pObj) { +static FORCE_INLINE void __cache_lock_destroy(SCacheObj *pCacheObj) { #if defined(LINUX) - pthread_rwlock_destroy(&pObj->lock); + pthread_rwlock_destroy(&pCacheObj->lock); #else - pthread_mutex_destroy(&pObj->lock); + pthread_mutex_destroy(&pCacheObj->lock); #endif } @@ -105,10 +105,10 @@ static SCacheDataNode *taosCreateHashNode(const char *key, size_t keyLen, const /** * addedTime object node into trash, and this object is closed for referencing if it is addedTime to trash * It will be removed until the pNode->refCount == 0 - * @param pObj Cache object + * @param pCacheObj Cache object * @param pNode Cache slot object */ -static void taosAddToTrash(SCacheObj *pObj, SCacheDataNode *pNode) { +static void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) { if (pNode->inTrash) { /* node is already in trash */ return; } @@ -116,31 +116,31 @@ static void taosAddToTrash(SCacheObj *pObj, SCacheDataNode *pNode) { STrashElem *pElem = calloc(1, sizeof(STrashElem)); pElem->pData = pNode; - pElem->next = pObj->pTrash; - if (pObj->pTrash) { - pObj->pTrash->prev = pElem; + pElem->next = pCacheObj->pTrash; + if (pCacheObj->pTrash) { + pCacheObj->pTrash->prev = pElem; } pElem->prev = NULL; - pObj->pTrash = pElem; + pCacheObj->pTrash = pElem; pNode->inTrash = true; - pObj->numOfElemsInTrash++; + pCacheObj->numOfElemsInTrash++; - pTrace("key:%s %p move to trash, numOfElem in trash:%d", pNode->key, pNode, pObj->numOfElemsInTrash); + pTrace("key:%s %p move to trash, numOfElem in trash:%d", pNode->key, pNode, pCacheObj->numOfElemsInTrash); } -static void taosRemoveFromTrash(SCacheObj *pObj, STrashElem *pElem) { +static void taosRemoveFromTrash(SCacheObj *pCacheObj, STrashElem *pElem) { if (pElem->pData->signature != (uint64_t)pElem->pData) { pError("key:sig:%d %p data has been released, ignore", pElem->pData->signature, pElem->pData); return; } - pObj->numOfElemsInTrash--; + pCacheObj->numOfElemsInTrash--; if (pElem->prev) { pElem->prev->next = pElem->next; } else { /* pnode is the header, update header */ - pObj->pTrash = pElem->next; + pCacheObj->pTrash = pElem->next; } if (pElem->next) { @@ -154,24 +154,24 @@ static void taosRemoveFromTrash(SCacheObj *pObj, STrashElem *pElem) { /** * remove nodes in trash with refCount == 0 in cache * @param pNode - * @param pObj + * @param pCacheObj * @param force force model, if true, remove data in trash without check refcount. * may cause corruption. So, forece model only applys before cache is closed */ -static void taosTrashEmpty(SCacheObj *pObj, bool force) { - __cache_wr_lock(pObj); +static void taosTrashEmpty(SCacheObj *pCacheObj, bool force) { + __cache_wr_lock(pCacheObj); - if (pObj->numOfElemsInTrash == 0) { - if (pObj->pTrash != NULL) { - pError("key:inconsistency data in cache, numOfElem in trash:%d", pObj->numOfElemsInTrash); + if (pCacheObj->numOfElemsInTrash == 0) { + if (pCacheObj->pTrash != NULL) { + pError("key:inconsistency data in cache, numOfElem in trash:%d", pCacheObj->numOfElemsInTrash); } - pObj->pTrash = NULL; + pCacheObj->pTrash = NULL; - __cache_unlock(pObj); + __cache_unlock(pCacheObj); return; } - STrashElem *pElem = pObj->pTrash; + STrashElem *pElem = pCacheObj->pTrash; while (pElem) { T_REF_VAL_CHECK(pElem->pData); @@ -181,50 +181,51 @@ static void taosTrashEmpty(SCacheObj *pObj, bool force) { if (force || (T_REF_VAL_GET(pElem->pData) == 0)) { pTrace("key:%s %p removed from trash. numOfElem in trash:%d", pElem->pData->key, pElem->pData, - pObj->numOfElemsInTrash - 1); + pCacheObj->numOfElemsInTrash - 1); STrashElem *p = pElem; pElem = pElem->next; - taosRemoveFromTrash(pObj, p); + taosRemoveFromTrash(pCacheObj, p); } else { pElem = pElem->next; } } - assert(pObj->numOfElemsInTrash >= 0); - __cache_unlock(pObj); + assert(pCacheObj->numOfElemsInTrash >= 0); + __cache_unlock(pCacheObj); } /** * release node - * @param pObj cache object + * @param pCacheObj cache object * @param pNode data node */ -static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pObj, SCacheDataNode *pNode) { +static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheDataNode *pNode) { if (pNode->signature != (uint64_t)pNode) { pError("key:%s, %p data is invalid, or has been released", pNode->key, pNode); return; } - taosHashRemove(pObj->pHashTable, pNode->key, pNode->keySize); - pTrace("key:%s is removed from cache,total:%d,size:%ldbytes", pNode->key, pObj->totalSize, pObj->totalSize); + int32_t size = pNode->size; + taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); + pTrace("key:%s is removed from cache,total:%d,size:%ldbytes", pNode->key, pCacheObj->totalSize, size); free(pNode); } /** * move the old node into trash - * @param pObj + * @param pCacheObj * @param pNode */ -static FORCE_INLINE void taosCacheMoveToTrash(SCacheObj *pObj, SCacheDataNode *pNode) { - taosHashRemove(pObj->pHashTable, pNode->key, pNode->keySize); - taosAddToTrash(pObj, pNode); +static FORCE_INLINE void taosCacheMoveToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) { + taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); + taosAddToTrash(pCacheObj, pNode); } /** * update data in cache - * @param pObj + * @param pCacheObj * @param pNode * @param key * @param keyLen @@ -232,7 +233,7 @@ static FORCE_INLINE void taosCacheMoveToTrash(SCacheObj *pObj, SCacheDataNode *p * @param dataSize * @return */ -static SCacheDataNode *taosUpdateCacheImpl(SCacheObj *pObj, SCacheDataNode *pNode, char *key, int32_t keyLen, +static SCacheDataNode *taosUpdateCacheImpl(SCacheObj *pCacheObj, SCacheDataNode *pNode, char *key, int32_t keyLen, void *pData, uint32_t dataSize, uint64_t duration) { SCacheDataNode *pNewNode = NULL; @@ -259,9 +260,9 @@ static SCacheDataNode *taosUpdateCacheImpl(SCacheObj *pObj, SCacheDataNode *pNod T_REF_INC(pNewNode); // the address of this node may be changed, so the prev and next element should update the corresponding pointer - taosHashPut(pObj->pHashTable, key, keyLen, &pNewNode, sizeof(void *)); + taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNewNode, sizeof(void *)); } else { - taosCacheMoveToTrash(pObj, pNode); + taosCacheMoveToTrash(pCacheObj, pNode); pNewNode = taosCreateHashNode(key, keyLen, pData, dataSize, duration); if (pNewNode == NULL) { @@ -271,7 +272,7 @@ static SCacheDataNode *taosUpdateCacheImpl(SCacheObj *pObj, SCacheDataNode *pNod T_REF_INC(pNewNode); // addedTime new element to hashtable - taosHashPut(pObj->pHashTable, key, keyLen, &pNewNode, sizeof(void *)); + taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNewNode, sizeof(void *)); } return pNewNode; @@ -282,12 +283,12 @@ static SCacheDataNode *taosUpdateCacheImpl(SCacheObj *pObj, SCacheDataNode *pNod * @param key * @param pData * @param size - * @param pObj + * @param pCacheObj * @param keyLen * @param pNode * @return */ -static FORCE_INLINE SCacheDataNode *taosAddToCacheImpl(SCacheObj *pObj, char *key, size_t keyLen, const void *pData, +static FORCE_INLINE SCacheDataNode *taosAddToCacheImpl(SCacheObj *pCacheObj, char *key, size_t keyLen, const void *pData, size_t dataSize, uint64_t duration) { SCacheDataNode *pNode = taosCreateHashNode(key, keyLen, pData, dataSize, duration); if (pNode == NULL) { @@ -295,70 +296,70 @@ static FORCE_INLINE SCacheDataNode *taosAddToCacheImpl(SCacheObj *pObj, char *ke } T_REF_INC(pNode); - taosHashPut(pObj->pHashTable, key, keyLen, &pNode, sizeof(void *)); + taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNode, sizeof(void *)); return pNode; } -static void doCleanupDataCache(SCacheObj *pObj) { - __cache_wr_lock(pObj); +static void doCleanupDataCache(SCacheObj *pCacheObj) { + __cache_wr_lock(pCacheObj); - if (taosHashGetSize(pObj->pHashTable) > 0) { - taosHashCleanup(pObj->pHashTable); + if (taosHashGetSize(pCacheObj->pHashTable) > 0) { + taosHashCleanup(pCacheObj->pHashTable); } - __cache_unlock(pObj); + __cache_unlock(pCacheObj); - taosTrashEmpty(pObj, true); - __cache_lock_destroy(pObj); + taosTrashEmpty(pCacheObj, true); + __cache_lock_destroy(pCacheObj); - memset(pObj, 0, sizeof(SCacheObj)); - free(pObj); + memset(pCacheObj, 0, sizeof(SCacheObj)); + free(pCacheObj); } /** - * refresh cache to remove data in both hash list and trash, if any nodes' refcount == 0, every pObj->refreshTime + * refresh cache to remove data in both hash list and trash, if any nodes' refcount == 0, every pCacheObj->refreshTime * @param handle Cache object handle */ static void taosCacheRefresh(void *handle, void *tmrId) { - SCacheObj *pObj = (SCacheObj *)handle; + SCacheObj *pCacheObj = (SCacheObj *)handle; - if (pObj == NULL || taosHashGetSize(pObj->pHashTable) == 0) { + if (pCacheObj == NULL || taosHashGetSize(pCacheObj->pHashTable) == 0) { pTrace("object is destroyed. no refresh retry"); return; } - if (pObj->deleting == 1) { - doCleanupDataCache(pObj); + if (pCacheObj->deleting == 1) { + doCleanupDataCache(pCacheObj); return; } uint64_t expiredTime = taosGetTimestampMs(); - pObj->statistics.refreshCount++; + pCacheObj->statistics.refreshCount++; - SHashMutableIterator *pIter = taosHashCreateIter(pObj->pHashTable); + SHashMutableIterator *pIter = taosHashCreateIter(pCacheObj->pHashTable); - __cache_wr_lock(pObj); + __cache_wr_lock(pCacheObj); while (taosHashIterNext(pIter)) { - if (pObj->deleting == 1) { + if (pCacheObj->deleting == 1) { taosHashDestroyIter(pIter); break; } SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter); if (pNode->expiredTime <= expiredTime && T_REF_VAL_GET(pNode) <= 0) { - taosCacheReleaseNode(pObj, pNode); + taosCacheReleaseNode(pCacheObj, pNode); } } - __cache_unlock(pObj); + __cache_unlock(pCacheObj); taosHashDestroyIter(pIter); - if (pObj->deleting == 1) { // clean up resources and abort - doCleanupDataCache(pObj); + if (pCacheObj->deleting == 1) { // clean up resources and abort + doCleanupDataCache(pCacheObj); } else { - taosTrashEmpty(pObj, false); - taosTmrReset(taosCacheRefresh, pObj->refreshTime, pObj, pObj->tmrCtrl, &pObj->pTimer); + taosTrashEmpty(pCacheObj, false); + taosTmrReset(taosCacheRefresh, pCacheObj->refreshTime, pCacheObj, pCacheObj->tmrCtrl, &pCacheObj->pTimer); } } @@ -367,104 +368,100 @@ SCacheObj *taosCacheInit(void *tmrCtrl, int64_t refreshTime) { return NULL; } - SCacheObj *pObj = (SCacheObj *)calloc(1, sizeof(SCacheObj)); - if (pObj == NULL) { + SCacheObj *pCacheObj = (SCacheObj *)calloc(1, sizeof(SCacheObj)); + if (pCacheObj == NULL) { pError("failed to allocate memory, reason:%s", strerror(errno)); return NULL; } - pObj->pHashTable = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false); - if (pObj->pHashTable == NULL) { - free(pObj); + pCacheObj->pHashTable = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false); + if (pCacheObj->pHashTable == NULL) { + free(pCacheObj); pError("failed to allocate memory, reason:%s", strerror(errno)); return NULL; } // set free cache node callback function for hash table - taosHashSetFreecb(pObj->pHashTable, taosFreeNode); + taosHashSetFreecb(pCacheObj->pHashTable, taosFreeNode); - pObj->refreshTime = refreshTime * 1000; - pObj->tmrCtrl = tmrCtrl; + pCacheObj->refreshTime = refreshTime * 1000; + pCacheObj->tmrCtrl = tmrCtrl; - taosTmrReset(taosCacheRefresh, pObj->refreshTime, pObj, pObj->tmrCtrl, &pObj->pTimer); + taosTmrReset(taosCacheRefresh, pCacheObj->refreshTime, pCacheObj, pCacheObj->tmrCtrl, &pCacheObj->pTimer); - if (__cache_lock_init(pObj) != 0) { - taosTmrStopA(&pObj->pTimer); - taosHashCleanup(pObj->pHashTable); - free(pObj); + if (__cache_lock_init(pCacheObj) != 0) { + taosTmrStopA(&pCacheObj->pTimer); + taosHashCleanup(pCacheObj->pHashTable); + free(pCacheObj); pError("failed to init lock, reason:%s", strerror(errno)); return NULL; } - return pObj; + return pCacheObj; } -void *taosCachePut(void *handle, char *key, char *pData, int dataSize, int duration) { +void *taosCachePut(SCacheObj *pCacheObj, char *key, void *pData, size_t dataSize, int duration) { SCacheDataNode *pNode; - SCacheObj * pObj; - pObj = (SCacheObj *)handle; - if (pObj == NULL || pObj->pHashTable == NULL) { + if (pCacheObj == NULL || pCacheObj->pHashTable == NULL) { return NULL; } size_t keyLen = strlen(key); - __cache_wr_lock(pObj); - SCacheDataNode **pt = (SCacheDataNode **)taosHashGet(pObj->pHashTable, key, keyLen); + __cache_wr_lock(pCacheObj); + SCacheDataNode **pt = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen); SCacheDataNode * pOld = (pt != NULL) ? (*pt) : NULL; if (pOld == NULL) { // do addedTime to cache - pNode = taosAddToCacheImpl(pObj, key, keyLen, pData, dataSize, duration * 1000L); + pNode = taosAddToCacheImpl(pCacheObj, key, keyLen, pData, dataSize, duration * 1000L); if (NULL != pNode) { - pTrace("key:%s %p added into cache, addedTime:%" PRIu64 ", expireTime:%" PRIu64 ", cache total:%d, size:%" PRId64 - " bytes, collision:%d", - key, pNode, pNode->addedTime, pNode->expiredTime, dataSize, pObj->totalSize, - pObj->statistics.numOfCollision); + pCacheObj->totalSize += pNode->size; + + pTrace("key:%s %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 ", total:%d, size:%" PRId64 " bytes", + key, pNode, pNode->addedTime, pNode->expiredTime, pCacheObj->totalSize, dataSize); } } else { // old data exists, update the node - pNode = taosUpdateCacheImpl(pObj, pOld, key, keyLen, pData, dataSize, duration * 1000L); + pNode = taosUpdateCacheImpl(pCacheObj, pOld, key, keyLen, pData, dataSize, duration * 1000L); pTrace("key:%s %p exist in cache, updated", key, pNode); } - __cache_unlock(pObj); + __cache_unlock(pCacheObj); return (pNode != NULL) ? pNode->data : NULL; } -void *taosCacheAcquireByName(void *handle, char *key) { - SCacheObj *pObj = (SCacheObj *)handle; - if (pObj == NULL || taosHashGetSize(pObj->pHashTable) == 0) { +void *taosCacheAcquireByName(SCacheObj *pCacheObj, const char *key) { + if (pCacheObj == NULL || taosHashGetSize(pCacheObj->pHashTable) == 0) { return NULL; } uint32_t keyLen = (uint32_t)strlen(key); - __cache_rd_lock(pObj); + __cache_rd_lock(pCacheObj); - SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGet(pObj->pHashTable, key, keyLen); + SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen); if (ptNode != NULL) { T_REF_INC(*ptNode); } - __cache_unlock(pObj); + __cache_unlock(pCacheObj); if (ptNode != NULL) { - atomic_add_fetch_32(&pObj->statistics.hitCount, 1); + atomic_add_fetch_32(&pCacheObj->statistics.hitCount, 1); pTrace("key:%s is retrieved from cache,refcnt:%d", key, T_REF_VAL_GET(*ptNode)); } else { - atomic_add_fetch_32(&pObj->statistics.missCount, 1); + atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1); pTrace("key:%s not in cache,retrieved failed", key); } - atomic_add_fetch_32(&pObj->statistics.totalAccess, 1); + atomic_add_fetch_32(&pCacheObj->statistics.totalAccess, 1); return (ptNode != NULL) ? (*ptNode)->data : NULL; } -void *taosCacheAcquireByData(void *handle, void *data) { - SCacheObj *pObj = (SCacheObj *)handle; - if (pObj == NULL || data == NULL) return NULL; +void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) { + if (pCacheObj == NULL || data == NULL) return NULL; size_t offset = offsetof(SCacheDataNode, data); SCacheDataNode *ptNode = (SCacheDataNode *)((char *)data - offset); @@ -482,9 +479,8 @@ void *taosCacheAcquireByData(void *handle, void *data) { return data; } -void *taosCacheTransfer(void *handle, void **data) { - SCacheObj *pObj = (SCacheObj *)handle; - if (pObj == NULL || data == NULL) return NULL; +void *taosCacheTransfer(SCacheObj *pCacheObj, void **data) { + if (pCacheObj == NULL || data == NULL) return NULL; size_t offset = offsetof(SCacheDataNode, data); SCacheDataNode *ptNode = (SCacheDataNode *)((char *)(*data) - offset); @@ -504,9 +500,8 @@ void *taosCacheTransfer(void *handle, void **data) { return d; } -void taosCacheRelease(void *handle, void **data, bool _remove) { - SCacheObj *pObj = (SCacheObj *)handle; - if (pObj == NULL || (*data) == NULL || (taosHashGetSize(pObj->pHashTable) + pObj->numOfElemsInTrash == 0)) { +void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { + if (pCacheObj == NULL || (*data) == NULL || (taosHashGetSize(pCacheObj->pHashTable) + pCacheObj->numOfElemsInTrash == 0)) { return; } @@ -522,13 +517,13 @@ void taosCacheRelease(void *handle, void **data, bool _remove) { *data = NULL; if (_remove) { - __cache_wr_lock(pObj); + __cache_wr_lock(pCacheObj); // pNode may be released immediately by other thread after the reference count of pNode is set to 0, // So we need to lock it in the first place. T_REF_DEC(pNode); - taosCacheMoveToTrash(pObj, pNode); + taosCacheMoveToTrash(pCacheObj, pNode); - __cache_unlock(pObj); + __cache_unlock(pCacheObj); } else { T_REF_DEC(pNode); } diff --git a/src/vnode/detail/inc/vnodeRead.h b/src/vnode/detail/inc/vnodeRead.h index 232a92608c37b54fdbfe0291facca818afcbbda8..4e6e04208d7fd08d4af0e59337683d157c068114 100644 --- a/src/vnode/detail/inc/vnodeRead.h +++ b/src/vnode/detail/inc/vnodeRead.h @@ -170,7 +170,7 @@ typedef struct SQueryRuntimeEnv { STSCursor cur; SQueryCostSummary summary; bool stableQuery; // is super table query or not - SQueryDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file + SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file /* * Temporarily hold the in-memory cache block info during scan cache blocks diff --git a/src/vnode/detail/src/vnodeQueryImpl.c b/src/vnode/detail/src/vnodeQueryImpl.c index 80e073567261e6572bf786646510effac54099d2..9eb3fb8b65f8f6299729b77edf4c14776f0d99f9 100644 --- a/src/vnode/detail/src/vnodeQueryImpl.c +++ b/src/vnode/detail/src/vnodeQueryImpl.c @@ -1532,7 +1532,7 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t return w; } -static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SQueryDiskbasedResultBuf *pResultBuf, int32_t sid, +static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t sid, int32_t numOfRowsPerPage) { if (pWindowRes->pos.pageId != -1) { return 0; @@ -1574,7 +1574,7 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SQueryDiskbasedR static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, int32_t sid, STimeWindow *win) { assert(win->skey <= win->ekey); - SQueryDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; + SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&win->skey, TSDB_KEYSIZE); if (pWindowRes == NULL) { @@ -2156,7 +2156,7 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat int32_t GROUPRESULTID = 1; - SQueryDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; + SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pData, bytes); if (pWindowRes == NULL) { @@ -5594,7 +5594,7 @@ void UNUSED_FUNC displayInterResult(SData **pdata, SQuery *pQuery, int32_t numOf } } -// static tFilePage *getMeterDataPage(SQueryDiskbasedResultBuf *pResultBuf, SMeterQueryInfo *pMeterQueryInfo, +// static tFilePage *getMeterDataPage(SDiskbasedResultBuf *pResultBuf, SMeterQueryInfo *pMeterQueryInfo, // int32_t index) { // SIDList pList = getDataBufPagesIdList(pResultBuf, pMeterQueryInfo->sid); // return getResultBufferPageById(pResultBuf, pList.pData[index]); @@ -5700,7 +5700,7 @@ void copyResToQueryResultBuf(STableQuerySupportObj *pSupporter, SQuery *pQuery) } SQueryRuntimeEnv * pRuntimeEnv = &pSupporter->runtimeEnv; - SQueryDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; + SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; int32_t id = getGroupResultId(pSupporter->subgroupIdx - 1); SIDList list = getDataBufPagesIdList(pResultBuf, pSupporter->offset + id); @@ -5883,7 +5883,7 @@ int32_t doMergeMetersResultsToGroupRes(STableQuerySupportObj *pSupporter, SQuery int32_t flushFromResultBuf(STableQuerySupportObj *pSupporter, const SQuery *pQuery, const SQueryRuntimeEnv *pRuntimeEnv) { - SQueryDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; + SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; int32_t capacity = (DEFAULT_INTERN_BUF_SIZE - sizeof(tFilePage)) / pQuery->rowSize; // the base value for group result, since the maximum number of table for each vnode will not exceed 100,000. diff --git a/src/vnode/tsdb/inc/tsdb.h b/src/vnode/tsdb/inc/tsdb.h index f6e97aefb0727885ec512fc71daee28d862289b1..e12f51fd44cb768ba530441705acce1f3e1a0cb6 100644 --- a/src/vnode/tsdb/inc/tsdb.h +++ b/src/vnode/tsdb/inc/tsdb.h @@ -19,9 +19,11 @@ #include #include +#include "dataformat.h" #include "taosdef.h" #include "taosmsg.h" -#include "dataformat.h" +#include "tarray.h" +#include "name.h" #ifdef __cplusplus extern "C" { @@ -181,23 +183,17 @@ int32_t tsdbInsertData(tsdb_repo_t *pRepo, SSubmitMsg *pMsg); // -- FOR QUERY TIME SERIES DATA -typedef void tsdb_query_handle_t; // Use void to hide implementation details - -// time window -typedef struct STimeWindow { - int64_t skey; - int64_t ekey; -} STimeWindow; +typedef void* tsdb_query_handle_t; // Use void to hide implementation details // typedef struct { // } SColumnFilterInfo; // query condition to build vnode iterator -typedef struct STSDBQueryCond { +typedef struct STsdbQueryCond { STimeWindow twindow; int32_t order; // desc/asc order to iterate the data block - SColumnFilterInfo colFilterInfo; -} STSDBQueryCond; + SColumnInfoEx colList; +} STsdbQueryCond; typedef struct SBlockInfo { STimeWindow window; @@ -209,15 +205,18 @@ typedef struct SBlockInfo { } SBlockInfo; // TODO: move this data struct out of the module -typedef struct SData { - int32_t num; - char * data; -} SData; +//typedef struct SData { +// int32_t num; +// char * data; +//} SData; -typedef struct SDataBlock { - int32_t numOfCols; - SData **pData; -} SDataBlock; +typedef struct SDataBlockInfo { + STimeWindow window; + int32_t size; + int32_t numOfCols; + int64_t uid; + int32_t sid; +} SDataBlockInfo; typedef struct STableIDList { STableId *tableIds; @@ -227,83 +226,107 @@ typedef struct STableIDList { typedef struct { } SFields; +#define TSDB_TS_GREATER_EQUAL 1 +#define TSDB_TS_LESS_EQUAL 2 + +typedef struct SQueryRowCond { + int32_t rel; + TSKEY ts; +} SQueryRowCond; + +typedef void *tsdbpos_t; + /** * Get the data block iterator, starting from position according to the query condition - * @param pRepo the TSDB repository to query on * @param pCond query condition, only includes the filter on primary time stamp * @param pTableList table sid list * @return */ -tsdb_query_handle_t *tsdbQueryFromTableID(tsdb_repo_t *pRepo, STSDBQueryCond *pCond, const STableIDList *pTableList); +tsdb_query_handle_t *tsdbQueryByTableId(STsdbQueryCond *pCond, SArray *idList, SArray *pColumnInfo); /** - * Get iterator for super tables, of which tags values satisfy the tag filter info - * - * NOTE: the tagFilterStr is an bin-expression for tag filter, such as ((tag_col = 5) and (tag_col2 > 7)) - * The filter string is sent from client directly. - * The build of the tags filter expression from string is done in the iterator generating function. - * - * @param pRepo the repository to query on - * @param pCond query condition - * @param pTagFilterStr tag filter info + * move to next block + * @param pQueryHandle * @return */ -tsdb_query_handle_t *tsdbQueryFromTagConds(tsdb_repo_t *pRepo, STSDBQueryCond *pCond, int16_t stableId, - const char *pTagFilterStr); +bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle); /** - * Reset to the start(end) position of current query, from which the iterator starts. + * Get current data block information * * @param pQueryHandle - * @param position set the iterator traverses position. (TSDB_POS_START|TSDB_POS_END) * @return */ -int32_t tsdbResetQuery(tsdb_query_handle_t *pQueryHandle, int16_t position); +SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle); /** - * move to next block - * @param pQueryHandle - * @param pCond + * + * Get the pre-calculated information w.r.t. current data block. + * + * In case of data block in cache, the pBlockStatis will always be NULL. + * If a block is not completed loaded from disk, the pBlockStatis will be NULL. + + * @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0 * @return */ -bool tsdbIterNext(tsdb_query_handle_t *pQueryHandle); +int32_t tsdbRetrieveDataBlockStatisInfo(tsdb_query_handle_t *pQueryHandle, SDataStatis **pBlockStatis); /** - * 当前数据块的信息,调用next函数后,只会获得block的信息,包括:行数、列数、skey/ekey信息。注意该信息并不是现在的SCompBlockInfo信息。 - * 因为SCompBlockInfo是完整的数据块信息,但是迭代器返回并不是。 - * 查询处理引擎会自己决定需要blockInfo, 还是预计算数据,亦或是完整的数据。 - * Get current data block information + * The query condition with primary timestamp is passed to iterator during its constructor function, + * the returned data block must be satisfied with the time window condition in any cases, + * which means the SData data block is not actually the completed disk data blocks. * * @param pQueryHandle * @return */ -SBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle); +SArray *tsdbRetrieveDataBlock(tsdb_query_handle_t *pQueryHandle, SArray *pIdList); /** - * 获取当前数据块的预计算信息,如果块不完整,无预计算信息,如果是cache块,无预计算信息。 + * todo remove the parameter of position, and order type * - * Get the pre-calculated information w.r.t. current data block. + * Reset to the start(end) position of current query, from which the iterator starts. * - * In case of data block in cache, the pBlockStatis will always be NULL. - * If a block is not completed loaded from disk, the pBlockStatis will be NULL. + * @param pQueryHandle + * @param position set the iterator traverses position + * @param order ascending order or descending order + * @return + */ +int32_t tsdbResetQuery(tsdb_query_handle_t *pQueryHandle, STimeWindow* window, tsdbpos_t position, int16_t order); - * @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0 +/** + * return the access position of current query handle + * @param pQueryHandle + * @return + */ +int32_t tsdbDataBlockSeek(tsdb_query_handle_t *pQueryHandle, tsdbpos_t pos); + +/** + * todo remove this function later + * @param pQueryHandle * @return */ -int32_t tsdbRetrieveDataBlockStatisInfo(tsdb_query_handle_t *pQueryHandle, SFields *pBlockStatis); +tsdbpos_t tsdbDataBlockTell(tsdb_query_handle_t *pQueryHandle); /** - * 返回加载到缓存中的数据,可能是磁盘数据也可能是内存数据,对客户透明。即使是磁盘数据,返回的结果也是磁盘块中,满足查询时间范围要求的数据行,并不是一个完整的磁盘数 - * 据块。 + * todo remove this function later + * @param pQueryHandle + * @param pIdList + * @return + */ +SArray *tsdbRetrieveDataRow(tsdb_query_handle_t *pQueryHandle, SArray *pIdList, SQueryRowCond *pCond); + +/** + * Get iterator for super tables, of which tags values satisfy the tag filter info * - * The query condition with primary timestamp is passed to iterator during its constructor function, - * the returned data block must be satisfied with the time window condition in any cases, - * which means the SData data block is not actually the completed disk data blocks. + * NOTE: the tagFilterStr is an bin-expression for tag filter, such as ((tag_col = 5) and (tag_col2 > 7)) + * The filter string is sent from client directly. + * The build of the tags filter expression from string is done in the iterator generating function. * - * @param pQueryHandle + * @param pCond query condition + * @param pTagFilterStr tag filter info * @return */ -SDataBlock *tsdbRetrieveDataBlock(tsdb_query_handle_t *pQueryHandle); +tsdb_query_handle_t *tsdbQueryFromTagConds(STsdbQueryCond *pCond, int16_t stableId, const char *pTagFilterStr); /** * Get the qualified tables for (super) table query. diff --git a/src/vnode/tsdb/src/tsdbMetaFile.c b/src/vnode/tsdb/src/tsdbMetaFile.c index 689f8033db895781ebfed49727c4f2b54b357969..2a32283c06dd50cf3637745816379e07f51f3b84 100644 --- a/src/vnode/tsdb/src/tsdbMetaFile.c +++ b/src/vnode/tsdb/src/tsdbMetaFile.c @@ -12,8 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include -#include +#include "os.h" #include "taosdef.h" #include "hash.h" diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..36593e719a81cd58e6bf758356a2f958373a6e42 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "tsdb.h" + diff --git a/tests/examples/c/demo.c b/tests/examples/c/demo.c index a0c4588e6f38a6c6e8f87ec2a984720833f3df3b..dc16185d9a1b74cad4492c09f48dc3437772e16e 100644 --- a/tests/examples/c/demo.c +++ b/tests/examples/c/demo.c @@ -35,6 +35,7 @@ int main(int argc, char *argv[]) { return 0; } + taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg"); // init TAOS taos_init(); @@ -44,8 +45,27 @@ int main(int argc, char *argv[]) { exit(1); } printf("success to connect to server\n"); - + int32_t code = taos_query(taos, "select * from test.t1"); + if (code != 0) { + printf("failed to execute query, reason:%s\n", taos_errstr(taos)); + } + + TAOS_RES* res = taos_use_result(taos); + TAOS_ROW row = NULL; + char buf[512] = {0}; + + int32_t numOfFields = taos_num_fields(res); + TAOS_FIELD* pFields = taos_fetch_fields(res); + + while((row = taos_fetch_row(res)) != NULL) { + taos_print_row(buf, row, pFields, numOfFields); + printf("%s\n", buf); + memset(buf, 0, 512); + } + + return 0; + taos_query(taos, "drop database demo"); if (taos_query(taos, "create database demo") != 0) { printf("failed to create database, reason:%s\n", taos_errstr(taos)); @@ -90,7 +110,7 @@ int main(int argc, char *argv[]) { exit(1); } - TAOS_ROW row; +// TAOS_ROW row; int rows = 0; int num_fields = taos_field_count(taos); TAOS_FIELD *fields = taos_fetch_fields(result);