diff --git a/include/common/common.h b/include/common/common.h index 0ae06e5a7f91406dbed933c9c5b95a18abcfbfdc..9b8a4654428eb19efdc727e228eff401fac848ed 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -62,6 +62,12 @@ typedef struct SConstantItem { SVariant value; } SConstantItem; +typedef struct { + uint32_t numOfTables; + SArray *pGroupList; + SHashObj *map; // speedup acquire the tableQueryInfo by table uid +} STableGroupInfo; + typedef struct SSDataBlock { SColumnDataAgg *pBlockAgg; SArray *pDataBlock; // SArray diff --git a/include/common/tmsg.h b/include/common/tmsg.h index fdf64b7af2bcaf19d8c08c4e700c13beef10078d..2aaa2168cc004fc88eb3fccbca5b248964461107 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -986,6 +986,14 @@ typedef struct { char msg[]; } SSubQueryMsg; +typedef struct { + SMsgHead header; + uint64_t sId; + uint64_t queryId; + uint64_t taskId; +} SSinkDataReq; + + typedef struct { SMsgHead header; uint64_t sId; diff --git a/include/dnode/vnode/tsdb/tsdb.h b/include/dnode/vnode/tsdb/tsdb.h index 3aeef4d01371ea2ff9c6150f1c073f89358f26b8..f3d9f7134682e52e2c90ba4f1d2a0a4902b22cfc 100644 --- a/include/dnode/vnode/tsdb/tsdb.h +++ b/include/dnode/vnode/tsdb/tsdb.h @@ -18,6 +18,7 @@ #include "mallocator.h" #include "meta.h" +#include "common.h" #ifdef __cplusplus extern "C" { @@ -39,6 +40,10 @@ typedef struct STable { STSchema *pSchema; } STable; +#define BLOCK_LOAD_OFFSET_SEQ_ORDER 1 +#define BLOCK_LOAD_TABLE_SEQ_ORDER 2 +#define BLOCK_LOAD_TABLE_RR_ORDER 3 + #define TABLE_TID(t) (t)->tid #define TABLE_UID(t) (t)->uid @@ -58,6 +63,22 @@ typedef struct STsdbCfg { int8_t compression; } STsdbCfg; +// query condition to build multi-table data block iterator +typedef struct STsdbQueryCond { + STimeWindow twindow; + int32_t order; // desc|asc order to iterate the data block + int32_t numOfCols; + SColumnInfo *colList; + bool loadExternalRows; // load external rows or not + int32_t type; // data block load type: +} STsdbQueryCond; + +typedef struct { + void *pTable; + TSKEY lastKey; + uint64_t uid; +} STableKeyInfo; + // STsdb STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta); void tsdbClose(STsdb *); @@ -70,6 +91,119 @@ int tsdbCommit(STsdb *pTsdb); int tsdbOptionsInit(STsdbCfg *); void tsdbOptionsClear(STsdbCfg *); +typedef void* tsdbReadHandleT; +/** + * Get the data block iterator, starting from position according to the query condition + * + * @param tsdb tsdb handle + * @param pCond query condition, including time window, result set order, and basic required columns for each block + * @param tableInfoGroup table object list in the form of set, grouped into different sets according to the + * group by condition + * @param qinfo query info handle from query processor + * @return + */ +tsdbReadHandleT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, + void *pRef); + +/** + * Get the last row of the given query time window for all the tables in STableGroupInfo object. + * Note that only one data block with only row will be returned while invoking retrieve data block function for + * all tables in this group. + * + * @param tsdb tsdb handle + * @param pCond query condition, including time window, result set order, and basic required columns for each block + * @param tableInfo table list. + * @return + */ +//tsdbReadHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, uint64_t qId, +// SMemRef *pRef); + + +tsdbReadHandleT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, void* pMemRef); + +bool isTsdbCacheLastRow(tsdbReadHandleT* pTsdbReadHandle); + +/** + * get num of rows in mem table + * + * @param pHandle + * @return row size + */ + +int64_t tsdbGetNumOfRowsInMemTable(tsdbReadHandleT* pHandle); + +/** + * move to next block if exists + * + * @param pTsdbReadHandle + * @return + */ +bool tsdbNextDataBlock(tsdbReadHandleT pTsdbReadHandle); + +/** + * Get current data block information + * + * @param pTsdbReadHandle + * @param pBlockInfo + * @return + */ +void tsdbRetrieveDataBlockInfo(tsdbReadHandleT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo); + +/** + * + * Get the pre-calculated information w.r.t. current data block. + * + * In case of data block in cache, the pBlockStatis will always be NULL. + * If a block is not completed loaded from disk, the pBlockStatis will be NULL. + + * @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0 + * @return + */ +int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReadHandleT *pTsdbReadHandle, SDataStatis **pBlockStatis); + +/** + * + * The query condition with primary timestamp is passed to iterator during its constructor function, + * the returned data block must be satisfied with the time window condition in any cases, + * which means the SData data block is not actually the completed disk data blocks. + * + * @param pTsdbReadHandle query handle + * @param pColumnIdList required data columns id list + * @return + */ +SArray *tsdbRetrieveDataBlock(tsdbReadHandleT *pTsdbReadHandle, SArray *pColumnIdList); + +/** + * destroy the created table group list, which is generated by tag query + * @param pGroupList + */ +void tsdbDestroyTableGroup(STableGroupInfo *pGroupList); + +/** + * create the table group result including only one table, used to handle the normal table query + * + * @param tsdb tsdbHandle + * @param uid table uid + * @param pGroupInfo the generated result + * @return + */ +int32_t tsdbGetOneTableGroup(STsdb *tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo); + +/** + * + * @param tsdb + * @param pTableIdList + * @param pGroupInfo + * @return + */ +int32_t tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo); + +/** + * clean up the query handle + * @param queryHandle + */ +void tsdbCleanupQueryHandle(tsdbReadHandleT queryHandle); + #ifdef __cplusplus } #endif diff --git a/source/libs/executor/inc/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h similarity index 77% rename from source/libs/executor/inc/dataSinkMgt.h rename to include/libs/executor/dataSinkMgt.h index d13423b25db6b5fcb8f1daca2aeeb91512cd0410..f13ba5f87e2051ef95782130fcacd286d4cee2ba 100644 --- a/source/libs/executor/inc/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -21,36 +21,38 @@ extern "C" { #endif #include "os.h" -#include "executorimpl.h" +#include "thash.h" +#include "executor.h" -#define DS_CAPACITY_ENOUGH 1 -#define DS_CAPACITY_FULL 2 -#define DS_NEED_SCHEDULE 3 -#define DS_END 4 -#define DS_IN_PROCESS 5 +#define DS_BUF_LOW 1 +#define DS_BUF_FULL 2 +#define DS_BUF_EMPTY 3 struct SDataSink; struct SSDataBlock; typedef struct SDataSinkMgtCfg { - uint32_t maxDataBlockNum; + uint32_t maxDataBlockNum; // todo: this should be numOfRows? uint32_t maxDataBlockNumPerQuery; } SDataSinkMgtCfg; int32_t dsDataSinkMgtInit(SDataSinkMgtCfg *cfg); -typedef void* DataSinkHandle; - typedef struct SInputData { - const SSDataBlock* pData; + const struct SSDataBlock* pData; SHashObj* pTableRetrieveTsMap; } SInputData; -typedef struct SOutPutData { +typedef struct SOutputData { int32_t numOfRows; int8_t compressed; char* pData; -} SOutPutData; + bool queryEnd; + bool needSchedule; + int32_t bufStatus; + int64_t useconds; + int8_t precision; +} SOutputData; /** * Create a subplan's datasinker handle for all later operations. @@ -66,16 +68,16 @@ int32_t dsCreateDataSinker(const struct SDataSink *pDataSink, DataSinkHandle* pH * @param pRes * @return error code */ -int32_t dsPutDataBlock(DataSinkHandle handle, const SInputData* pInput, int32_t* pStatus); +int32_t dsPutDataBlock(DataSinkHandle handle, const SInputData* pInput, bool* pContinue); -void dsEndPut(DataSinkHandle handle); +void dsEndPut(DataSinkHandle handle, int64_t useconds); /** * Get the length of the data returned by the next call to dsGetDataBlock. * @param handle - * @return data length + * @param pLen data length */ -int32_t dsGetDataLength(DataSinkHandle handle, int32_t* pStatus); +void dsGetDataLength(DataSinkHandle handle, int32_t* pLen, bool* pQueryEnd); /** * Get data, the caller needs to allocate data memory. @@ -84,7 +86,7 @@ int32_t dsGetDataLength(DataSinkHandle handle, int32_t* pStatus); * @param pStatus output * @return error code */ -int32_t dsGetDataBlock(DataSinkHandle handle, SOutPutData* pOutput, int32_t* pStatus); +int32_t dsGetDataBlock(DataSinkHandle handle, SOutputData* pOutput); /** * After dsGetStatus returns DS_NEED_SCHEDULE, the caller need to put this into the work queue. diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 17c11b5d09215cf59b8ad8e6d5b085282f07ff7a..71b014d02565a8c3331abfce112c0d8b1bc0f293 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -21,24 +21,30 @@ extern "C" { #endif typedef void* qTaskInfo_t; +typedef void* DataSinkHandle; +struct SSubplan; -/** - * create the qinfo object according to QueryTableMsg - * @param tsdb - * @param pQueryTableMsg - * @param pTaskInfo - * @return - */ -int32_t qCreateTask(void* tsdb, int32_t vgId, void* pQueryTableMsg, qTaskInfo_t* pTaskInfo, uint64_t qId); + + /** + * Create the exec task object according to task json + * @param tsdb + * @param vgId + * @param pTaskInfoMsg + * @param pTaskInfo + * @param qId + * @return + */ +int32_t qCreateExecTask(void* tsdb, int32_t vgId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo); /** - * the main query execution function, including query on both table and multiple tables, + * The main task execution function, including query on both table and multiple tables, * which are decided according to the tag or table name query conditions * - * @param qinfo + * @param tinfo + * @param handle * @return */ -bool qExecTask(qTaskInfo_t qinfo, uint64_t *qId); +int32_t qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle); /** * Retrieve the produced results information, if current query is not paused or completed, @@ -60,7 +66,7 @@ int32_t qRetrieveQueryResultInfo(qTaskInfo_t qinfo, bool* buildRes, void* pRspCo * @param contLen payload length * @return */ -int32_t qDumpRetrieveResult(qTaskInfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec); +//int32_t qDumpRetrieveResult(qTaskInfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec); /** * return the transporter context (RPC) @@ -81,7 +87,7 @@ int32_t qKillTask(qTaskInfo_t qinfo); * @param qinfo * @return */ -int32_t qIsQueryCompleted(qTaskInfo_t qinfo); +int32_t qIsTaskCompleted(qTaskInfo_t qinfo); /** * destroy query info structure @@ -113,7 +119,7 @@ int32_t qGetQualifiedTableIdList(void* pTableList, const char* tagCond, int32_t * @param numOfIndex * @return */ -int32_t qCreateTableGroupByGroupExpr(SArray* pTableIdList, TSKEY skey, STableGroupInfo groupInfo, SColIndex* groupByIndex, int32_t numOfIndex); +//int32_t qCreateTableGroupByGroupExpr(SArray* pTableIdList, TSKEY skey, STableGroupInfo groupInfo, SColIndex* groupByIndex, int32_t numOfIndex); /** * Update the table id list of a given query. diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 74b7813465659479edd343238b45a5b9917b18ee..aa87dd155b139977483a92a02d2b8f9fa950aed7 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -20,8 +20,10 @@ extern "C" { #endif -#include "planner.h" #include "catalog.h" +#include "planner.h" + +struct SSchJob; typedef struct SSchedulerCfg { uint32_t maxJobNum; @@ -65,7 +67,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg); * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob, SQueryResult *pRes); +int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes); /** * Process the query job, generated according to the query physical plan. @@ -73,7 +75,7 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob); +int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob); /** * Fetch query result from the remote query executor @@ -81,7 +83,7 @@ int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, * @param data * @return */ -int32_t scheduleFetchRows(void *pJob, void **data); +int32_t scheduleFetchRows(struct SSchJob *pJob, void **data); /** @@ -89,7 +91,7 @@ int32_t scheduleFetchRows(void *pJob, void **data); * @param pJob * @return */ -int32_t scheduleCancelJob(void *pJob); +//int32_t scheduleCancelJob(void *pJob); /** * Free the query job diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 26afe237c9f4e42c1b1b7b16fb34d50942f2852e..7292375e7fd1f839f309c49c772438eb66e7ce97 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -113,6 +113,7 @@ typedef struct SRequestSendRecvBody { tsem_t rspSem; // not used now void* fp; SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed. + struct SSchJob *pQueryJob; // query job, created according to sql query DAG. SDataBuf requestMsg; SReqResultInfo resInfo; } SRequestSendRecvBody; @@ -129,7 +130,7 @@ typedef struct SRequestObj { char *msgBuf; void *pInfo; // sql parse info, generated by parser module int32_t code; - uint64_t affectedRows; + uint64_t affectedRows; // todo remove it SQueryExecMetric metric; SRequestSendRecvBody body; } SRequestObj; @@ -161,12 +162,13 @@ int taos_options_imp(TSDB_OPTION option, const char *str); void* openTransporter(const char *user, const char *auth, int32_t numOfThreads); void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); + void initMsgHandleFp(); TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port); -TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen); void *doFetchRow(SRequestObj* pRequest); + void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); #ifdef __cplusplus diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d18142cebfc4a6a0bbb59f2bdb8296e3bf0bf5ad..727b668f3adb138a814facbb2489401a48e2551a 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1,6 +1,10 @@ +#include "../../libs/scheduler/inc/schedulerInt.h" #include "clientInt.h" #include "clientLog.h" +#include "parser.h" +#include "planner.h" +#include "scheduler.h" #include "tdef.h" #include "tep.h" #include "tglobal.h" @@ -8,9 +12,6 @@ #include "tnote.h" #include "tpagedfile.h" #include "tref.h" -#include "parser.h" -#include "planner.h" -#include "scheduler.h" #define CHECK_CODE_GOTO(expr, lable) \ do { \ @@ -57,6 +58,7 @@ static char* getClusterKey(const char* user, const char* auth, const char* ip, i } static STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo); +static void setResSchemaInfo(SReqResultInfo* pResInfo, const SDataBlockSchema* pDataBlockSchema); TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port) { if (taos_init() != TSDB_CODE_SUCCESS) { @@ -197,19 +199,49 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) { int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag) { pRequest->type = pQueryNode->type; - return qCreateQueryDag(pQueryNode, pDag, pRequest->requestId); + + SReqResultInfo* pResInfo = &pRequest->body.resInfo; + int32_t code = qCreateQueryDag(pQueryNode, pDag, pRequest->requestId); + if (code != 0) { + return code; + } + + if (pQueryNode->type == TSDB_SQL_SELECT) { + SArray* pa = taosArrayGetP((*pDag)->pSubplans, 0); + + SSubplan* pPlan = taosArrayGetP(pa, 0); + SDataBlockSchema* pDataBlockSchema = &(pPlan->pDataSink->schema); + setResSchemaInfo(pResInfo, pDataBlockSchema); + pRequest->type = TDMT_VND_QUERY; + } + + return code; +} + +void setResSchemaInfo(SReqResultInfo* pResInfo, const SDataBlockSchema* pDataBlockSchema) { + assert(pDataBlockSchema != NULL && pDataBlockSchema->numOfCols > 0); + + pResInfo->numOfCols = pDataBlockSchema->numOfCols; + pResInfo->fields = calloc(pDataBlockSchema->numOfCols, sizeof(pDataBlockSchema->pSchema[0])); + + for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { + SSchema* pSchema = &pDataBlockSchema->pSchema[i]; + pResInfo->fields[i].bytes = pSchema->bytes; + pResInfo->fields[i].type = pSchema->type; + tstrncpy(pResInfo->fields[i].name, pSchema[i].name, tListLen(pResInfo->fields[i].name)); + } } -int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, void** pJob) { +int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) { if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) { SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; - int32_t code = scheduleExecJob(pRequest->pTscObj->pTransporter, NULL, pDag, pJob, &res); + int32_t code = scheduleExecJob(pRequest->pTscObj->pTransporter, NULL, pDag, &pRequest->body.pQueryJob, &res); if (code != TSDB_CODE_SUCCESS) { // handle error and retry } else { - if (*pJob != NULL) { - scheduleFreeJob(*pJob); + if (pRequest->body.pQueryJob != NULL) { + scheduleFreeJob(pRequest->body.pQueryJob); } } @@ -217,7 +249,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, void** pJob) { return res.code; } - return scheduleAsyncExecJob(pRequest->pTscObj->pTransporter, NULL /*todo appInfo.xxx*/, pDag, pJob); + return scheduleAsyncExecJob(pRequest->pTscObj->pTransporter, NULL, pDag, &pRequest->body.pQueryJob); } TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen) { @@ -294,7 +326,6 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { SRequestObj *pRequest = NULL; SQueryNode *pQuery = NULL; SQueryDag *pDag = NULL; - void *pJob = NULL; terrno = TSDB_CODE_SUCCESS; CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); @@ -304,9 +335,8 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return); } else { CHECK_CODE_GOTO(getPlan(pRequest, pQuery, &pDag), _return); - CHECK_CODE_GOTO(scheduleQuery(pRequest, pDag, &pJob), _return); + CHECK_CODE_GOTO(scheduleQuery(pRequest, pDag), _return); pRequest->code = terrno; - return pRequest; } _return: @@ -315,6 +345,7 @@ _return: if (NULL != pRequest && TSDB_CODE_SUCCESS != terrno) { pRequest->code = terrno; } + return pRequest; } @@ -513,7 +544,17 @@ void* doFetchRow(SRequestObj* pRequest) { SReqResultInfo* pResultInfo = &pRequest->body.resInfo; if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) { - if (pRequest->type == TDMT_MND_SHOW) { + if (pRequest->type == TDMT_VND_QUERY) { + pRequest->type = TDMT_VND_FETCH; + scheduleFetchRows(pRequest->body.pQueryJob, (void **)&pRequest->body.resInfo.pData); + + pResultInfo->current = 0; + if (pResultInfo->numOfRows <= pResultInfo->current) { + return NULL; + } + + goto _return; + } else if (pRequest->type == TDMT_MND_SHOW) { pRequest->type = TDMT_MND_SHOW_RETRIEVE; } else if (pRequest->type == TDMT_VND_SHOW_TABLES) { pRequest->type = TDMT_VND_SHOW_TABLES_FETCH; @@ -556,6 +597,8 @@ void* doFetchRow(SRequestObj* pRequest) { } } +_return: + for(int32_t i = 0; i < pResultInfo->numOfCols; ++i) { pResultInfo->row[i] = pResultInfo->pCol[i] + pResultInfo->fields[i].bytes * pResultInfo->current; if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 73c9fc5e9f12b1cdcec189c07efc7c127bb06df7..d507c565df07beaac5102b9553a3b7da71733b47 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -24,7 +24,6 @@ #include "../inc/clientInt.h" #include "taos.h" -#include "tglobal.h" namespace { void showDB(TAOS* pConn) { @@ -57,449 +56,449 @@ TEST(testCase, connect_Test) { taos_close(pConn); } -TEST(testCase, create_user_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, create_account_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, drop_account_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_user_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "show users"); - TAOS_ROW pRow = NULL; - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, drop_user_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "drop user abc"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_db_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "show databases"); - TAOS_ROW pRow = NULL; - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_close(pConn); -} - -TEST(testCase, create_db_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - - pRes = taos_query(pConn, "create database abc1 vgroups 4"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } - taos_close(pConn); -} - -TEST(testCase, create_dnode_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); - if (taos_errno(pRes) != 0) { - printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); - if (taos_errno(pRes) != 0) { - printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - taos_close(pConn); -} - -TEST(testCase, drop_dnode_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "drop dnode 2"); - if (taos_errno(pRes) != 0) { - printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, use_db_test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_close(pConn); -} - -// TEST(testCase, drop_db_test) { +//TEST(testCase, create_user_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, create_account_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, drop_account_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, show_user_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "show users"); +// TAOS_ROW pRow = NULL; +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, drop_user_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "drop user abc"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, show_db_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "show databases"); +// TAOS_ROW pRow = NULL; +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_close(pConn); +//} +// +//TEST(testCase, create_db_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); +// if (taos_errno(pRes) != 0) { +// printf("error in create db, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create database abc1 vgroups 4"); +// if (taos_errno(pRes) != 0) { +// printf("error in create db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_close(pConn); +//} +// +//TEST(testCase, create_dnode_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); +// if (taos_errno(pRes) != 0) { +// printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// taos_close(pConn); +//} +// +//TEST(testCase, drop_dnode_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "drop dnode 2"); +// if (taos_errno(pRes) != 0) { +// printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, use_db_test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in use db, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_close(pConn); +//} +// +//// TEST(testCase, drop_db_test) { +//// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +//// assert(pConn != NULL); +//// +//// showDB(pConn); +//// +//// TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); +//// if (taos_errno(pRes) != 0) { +//// printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); +//// } +//// taos_free_result(pRes); +//// +//// showDB(pConn); +//// +//// pRes = taos_query(pConn, "create database abc1"); +//// if (taos_errno(pRes) != 0) { +//// printf("create to drop db, reason:%s\n", taos_errstr(pRes)); +//// } +//// taos_free_result(pRes); +//// taos_close(pConn); +////} +// +//TEST(testCase, create_stable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); +// if (taos_errno(pRes) != 0) { +// printf("error in create db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)"); +// if (taos_errno(pRes) != 0) { +// printf("error in create stable, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, create_table_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table tm0(ts timestamp, k int)"); +// taos_free_result(pRes); +// +// taos_close(pConn); +//} +// +//TEST(testCase, create_ctable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, show_stable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "show stables"); +// if (taos_errno(pRes) != 0) { +// printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, show_vgroup_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "show vgroups"); +// if (taos_errno(pRes) != 0) { +// printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, create_multiple_tables) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// ASSERT_NE(pConn, nullptr); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// taos_close(pConn); +// return; +// } +// +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table t_2 using st1 tags(1)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// taos_free_result(pRes); +// pRes = taos_query(pConn, "create table t_3 using st1 tags(2)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// +// for (int32_t i = 0; i < 20; ++i) { +// char sql[512] = {0}; +// snprintf(sql, tListLen(sql), +// "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i, +// (i + 1) * 30, (i + 2) * 40); +// TAOS_RES* pres = taos_query(pConn, sql); +// if (taos_errno(pres) != 0) { +// printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres)); +// } +// taos_free_result(pres); +// } +// +// taos_close(pConn); +//} +// +//TEST(testCase, show_table_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // assert(pConn != NULL); // -// showDB(pConn); +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "show tables"); +// if (taos_errno(pRes) != 0) { +// printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, drop_stable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); // -// TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); +// TAOS_RES* pRes = taos_query(pConn, "create database abc1"); // if (taos_errno(pRes) != 0) { -// printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); +// printf("error in creating db, reason:%s\n", taos_errstr(pRes)); // } // taos_free_result(pRes); // -// showDB(pConn); +// pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in using db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); // -// pRes = taos_query(pConn, "create database abc1"); +// pRes = taos_query(pConn, "drop stable st1"); // if (taos_errno(pRes) != 0) { -// printf("create to drop db, reason:%s\n", taos_errstr(pRes)); +// printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); // } +// // taos_free_result(pRes); // taos_close(pConn); //} - -TEST(testCase, create_stable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)"); - if (taos_errno(pRes) != 0) { - printf("error in create stable, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, create_table_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table tm0(ts timestamp, k int)"); - taos_free_result(pRes); - - taos_close(pConn); -} - -TEST(testCase, create_ctable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_stable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "show stables"); - if (taos_errno(pRes) != 0) { - printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_vgroup_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "show vgroups"); - if (taos_errno(pRes) != 0) { - printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, create_multiple_tables) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - ASSERT_NE(pConn, nullptr); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s", taos_errstr(pRes)); - taos_free_result(pRes); - taos_close(pConn); - return; - } - - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table t_2 using st1 tags(1)"); - if (taos_errno(pRes) != 0) { - printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - taos_free_result(pRes); - pRes = taos_query(pConn, "create table t_3 using st1 tags(2)"); - if (taos_errno(pRes) != 0) { - printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - - for (int32_t i = 0; i < 20; ++i) { - char sql[512] = {0}; - snprintf(sql, tListLen(sql), - "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i, - (i + 1) * 30, (i + 2) * 40); - TAOS_RES* pres = taos_query(pConn, sql); - if (taos_errno(pres) != 0) { - printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres)); - } - taos_free_result(pres); - } - - taos_close(pConn); -} - -TEST(testCase, show_table_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - taos_free_result(pRes); - - pRes = taos_query(pConn, "show tables"); - if (taos_errno(pRes) != 0) { - printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, drop_stable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create database abc1"); - if (taos_errno(pRes) != 0) { - printf("error in creating db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in using db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "drop stable st1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, generated_request_id_test) { - SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); - - for (int32_t i = 0; i < 50000; ++i) { - uint64_t v = generateRequestId(); - void* result = taosHashGet(phash, &v, sizeof(v)); - if (result != nullptr) { - printf("0x%lx, index:%d\n", v, i); - } - assert(result == nullptr); - taosHashPut(phash, &v, sizeof(v), NULL, 0); - } - - taosHashCleanup(phash); -} +// +//TEST(testCase, generated_request_id_test) { +// SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); +// +// for (int32_t i = 0; i < 50000; ++i) { +// uint64_t v = generateRequestId(); +// void* result = taosHashGet(phash, &v, sizeof(v)); +// if (result != nullptr) { +// printf("0x%lx, index:%d\n", v, i); +// } +// assert(result == nullptr); +// taosHashPut(phash, &v, sizeof(v), NULL, 0); +// } +// +// taosHashCleanup(phash); +//} // TEST(testCase, create_topic_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -557,18 +556,29 @@ TEST(testCase, projection_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); - TAOS_RES* pRes = taos_query(pConn, "use test1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s", taos_errstr(pRes)); - taos_free_result(pRes); - return; - } +// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// return; +// } +// taos_free_result(pRes); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + +// pRes = taos_query(pConn, "create table m1 (ts timestamp, k int) tags(a int)"); taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table tu using m1 tags(1)"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "insert into tu values(now, 1)"); +// taos_free_result(pRes); - pRes = taos_query(pConn, "select * from tm0"); + pRes = taos_query(pConn, "select * from tu"); if (taos_errno(pRes) != 0) { - printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); + printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); ASSERT_TRUE(false); } diff --git a/source/dnode/vnode/impl/src/vnodeQuery.c b/source/dnode/vnode/impl/src/vnodeQuery.c index 29b6984937e3171c2b6447d63e4ff3017139bca6..909b233efb9a6415ecb35175210db20ebfc7847a 100644 --- a/source/dnode/vnode/impl/src/vnodeQuery.c +++ b/source/dnode/vnode/impl/src/vnodeQuery.c @@ -23,7 +23,7 @@ int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NULL, &pVnode->pQuery); int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { vTrace("query message is processed"); - return qWorkerProcessQueryMsg(pVnode, pVnode->pQuery, pMsg); + return qWorkerProcessQueryMsg(pVnode->pTsdb, pVnode->pQuery, pMsg); } int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { diff --git a/source/dnode/vnode/tsdb/CMakeLists.txt b/source/dnode/vnode/tsdb/CMakeLists.txt index 65dc95a9d87e1a17cd35b218d3b156f3c9fdb7eb..e38ba1c466fee560253d27bd38de5b36c1b8437c 100644 --- a/source/dnode/vnode/tsdb/CMakeLists.txt +++ b/source/dnode/vnode/tsdb/CMakeLists.txt @@ -13,6 +13,7 @@ else(0) "src/tsdbReadImpl.c" "src/tsdbFile.c" "src/tsdbFS.c" + "src/tsdbRead.c" ) endif(0) diff --git a/source/dnode/vnode/tsdb/src/tsdbCommit.c b/source/dnode/vnode/tsdb/src/tsdbCommit.c index 2be032a3354772355ce5fa7130e24df18f146d69..5aab343c4b1435f2571c7922598ef170d6b67d9c 100644 --- a/source/dnode/vnode/tsdb/src/tsdbCommit.c +++ b/source/dnode/vnode/tsdb/src/tsdbCommit.c @@ -1253,7 +1253,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols * pBlock->keyFirst = dataColsKeyFirst(pDataCols); pBlock->keyLast = dataColsKeyLast(pDataCols); - tsdbDebug("vgId:%d tid:%d a block of data is written to file %s, offset %" PRId64 + tsdbDebug("vgId:%d uid:%"PRId64" a block of data is written to file %s, offset %" PRId64 " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, REPO_ID(pRepo), TABLE_TID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len, pBlock->numOfCols, pBlock->keyFirst, pBlock->keyLast); diff --git a/source/dnode/vnode/tsdb/src/tsdbRead.c b/source/dnode/vnode/tsdb/src/tsdbRead.c index da5481dae184b1cc0aa280f9e3d4a952571abfe1..906046ed9a11729deca7f353b1bf5fd4b5510c7d 100644 --- a/source/dnode/vnode/tsdb/src/tsdbRead.c +++ b/source/dnode/vnode/tsdb/src/tsdbRead.c @@ -13,18 +13,23 @@ * along with this program. If not, see . */ +#include "tsdb.h" +#include "tsdbDef.h" +#include "tsdbFS.h" +#include "tsdbLog.h" +#include "tsdbReadImpl.h" +#include "ttime.h" +#include "exception.h" #include "os.h" -#include "tdataformat.h" -#include "tskiplist.h" -#include "tulog.h" #include "talgo.h" #include "tcompare.h" -#include "exception.h" +#include "tdataformat.h" +#include "tskiplist.h" #include "taosdef.h" #include "tlosertree.h" #include "tsdbint.h" -#include "texpr.h" +#include "tmsg.h" #define EXTRA_BYTES 2 #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) @@ -34,8 +39,7 @@ ((SDataBlockInfo){.window = {.skey = (_block)->keyFirst, .ekey = (_block)->keyLast}, \ .numOfCols = (_block)->numOfCols, \ .rows = (_block)->numOfRows, \ - .tid = (_checkInfo)->tableId.tid, \ - .uid = (_checkInfo)->tableId.uid}) + .uid = (_checkInfo)->tableId}) enum { TSDB_QUERY_TYPE_ALL = 1, @@ -62,7 +66,7 @@ typedef struct SQueryFilePos { typedef struct SDataBlockLoadInfo { SDFileSet* fileGroup; int32_t slot; - int32_t tid; + uint64_t uid; SArray* pLoadedCols; } SDataBlockLoadInfo; @@ -79,13 +83,13 @@ enum { typedef struct STableCheckInfo { - STableId tableId; + uint64_t tableId; TSKEY lastKey; STable* pTableObj; SBlockInfo* pCompInfo; int32_t compSize; int32_t numOfBlocks:29; // number of qualified data blocks not the original blocks - uint8_t chosen:2; // indicate which iterator should move forward + uint8_t chosen:2; // indicate which iterator should move forward bool initBuf; // whether to initialize the in-memory skip list iterator or not SSkipListIterator* iter; // mem buffer skip list iterator SSkipListIterator* iiter; // imem buffer skip list iterator @@ -111,8 +115,8 @@ typedef struct SIOCostSummary { int64_t headFileLoadTime; } SIOCostSummary; -typedef struct STsdbQueryHandle { - STsdbRepo* pTsdb; +typedef struct STsdbReadHandle { + STsdb* pTsdb; SQueryFilePos cur; // current position int16_t order; STimeWindow window; // the primary query time window that applies to all queries @@ -137,7 +141,8 @@ typedef struct STsdbQueryHandle { STableBlockInfo* pDataBlockInfo; SDataCols *pDataCols; // in order to hold current file data block int32_t allocSize; // allocated data block size - SMemRef *pMemRef; +// STsdb +// STsdbMemTable * pMemTable; SArray *defaultLoadColumn;// default load column SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQueryAttr */ @@ -145,7 +150,7 @@ typedef struct STsdbQueryHandle { SArray *prev; // previous row which is before than time window SArray *next; // next row which is after the query time window SIOCostSummary cost; -} STsdbQueryHandle; +} STsdbReadHandle; typedef struct STableGroupSupporter { int32_t numOfCols; @@ -154,23 +159,23 @@ typedef struct STableGroupSupporter { } STableGroupSupporter; static STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList); -static int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList); -static int32_t checkForCachedLast(STsdbQueryHandle* pQueryHandle); -static int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey); +static int32_t checkForCachedLastRow(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo *groupList); +static int32_t checkForCachedLast(STsdbReadHandle* pTsdbReadHandle); +//static int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey); -static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); -static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock); +static void changeQueryHandleForInterpQuery(tsdbReadHandleT pHandle); +static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock); static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); -static int32_t tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, STsdbQueryHandle* pQueryHandle); +static int32_t tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, STsdbReadHandle* pTsdbReadHandle); static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2); -static int32_t doGetExternalRow(STsdbQueryHandle* pQueryHandle, int16_t type, SMemRef* pMemRef); -static void* doFreeColumnInfoData(SArray* pColumnInfoData); -static void* destroyTableCheckInfo(SArray* pTableCheckInfo); -static bool tsdbGetExternalRow(TsdbQueryHandleT pHandle); +//static int32_t doGetExternalRow(STsdbReadHandle* pTsdbReadHandle, int16_t type, void* pMemRef); +//static void* doFreeColumnInfoData(SArray* pColumnInfoData); +//static void* destroyTableCheckInfo(SArray* pTableCheckInfo); +static bool tsdbGetExternalRow(tsdbReadHandleT pHandle); static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { pBlockLoadInfo->slot = -1; - pBlockLoadInfo->tid = -1; + pBlockLoadInfo->uid = 0; pBlockLoadInfo->fileGroup = NULL; } @@ -179,21 +184,21 @@ static void tsdbInitCompBlockLoadInfo(SLoadCompBlockInfo* pCompBlockLoadInfo) { pCompBlockLoadInfo->fileId = -1; } -static SArray* getColumnIdList(STsdbQueryHandle* pQueryHandle) { - size_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); +static SArray* getColumnIdList(STsdbReadHandle* pTsdbReadHandle) { + size_t numOfCols = QH_GET_NUM_OF_COLS(pTsdbReadHandle); assert(numOfCols <= TSDB_MAX_COLUMNS); SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t)); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pColumns, i); taosArrayPush(pIdList, &pCol->info.colId); } return pIdList; } -static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS) { - SArray* pLocalIdList = getColumnIdList(pQueryHandle); +static SArray* getDefaultLoadColumns(STsdbReadHandle* pTsdbReadHandle, bool loadTS) { + SArray* pLocalIdList = getColumnIdList(pTsdbReadHandle); // check if the primary time stamp column needs to load int16_t colId = *(int16_t*)taosArrayGet(pLocalIdList, 0); @@ -207,63 +212,63 @@ static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS return pLocalIdList; } -static void tsdbMayTakeMemSnapshot(STsdbQueryHandle* pQueryHandle, SArray* psTable) { - assert(pQueryHandle != NULL && pQueryHandle->pMemRef != NULL); - - SMemRef* pMemRef = pQueryHandle->pMemRef; - if (pQueryHandle->pMemRef->ref++ == 0) { - tsdbTakeMemSnapshot(pQueryHandle->pTsdb, &(pMemRef->snapshot), psTable); - } - - taosArrayDestroy(psTable); -} - -static void tsdbMayUnTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) { - assert(pQueryHandle != NULL); - SMemRef* pMemRef = pQueryHandle->pMemRef; - if (pMemRef == NULL) { // it has been freed - return; - } - - if (--pMemRef->ref == 0) { - tsdbUnTakeMemSnapShot(pQueryHandle->pTsdb, &(pMemRef->snapshot)); - } - - pQueryHandle->pMemRef = NULL; +static void tsdbMayTakeMemSnapshot(STsdbReadHandle* pTsdbReadHandle, SArray* psTable) { +// assert(pTsdbReadHandle != NULL && pTsdbReadHandle->pMemRef != NULL); +// +// STsdbMemTable* pMemRef = pTsdbReadHandle->pMemRef; +// if (pTsdbReadHandle->pMemRef->ref++ == 0) { +// tsdbTakeMemSnapshot(pTsdbReadHandle->pTsdb, &(pMemRef->snapshot), psTable); +// } +// +// taosArrayDestroy(psTable); } -int64_t tsdbGetNumOfRowsInMemTable(TsdbQueryHandleT* pHandle) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; - - int64_t rows = 0; - SMemRef* pMemRef = pQueryHandle->pMemRef; - if (pMemRef == NULL) { return rows; } - - STableData* pMem = NULL; - STableData* pIMem = NULL; - - SMemTable* pMemT = pMemRef->snapshot.mem; - SMemTable* pIMemT = pMemRef->snapshot.imem; - - size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - for (int32_t i = 0; i < size; ++i) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); - - if (pMemT && pCheckInfo->tableId.tid < pMemT->maxTables) { - pMem = pMemT->tData[pCheckInfo->tableId.tid]; - rows += (pMem && pMem->uid == pCheckInfo->tableId.uid) ? pMem->numOfRows : 0; - } - if (pIMemT && pCheckInfo->tableId.tid < pIMemT->maxTables) { - pIMem = pIMemT->tData[pCheckInfo->tableId.tid]; - rows += (pIMem && pIMem->uid == pCheckInfo->tableId.uid) ? pIMem->numOfRows : 0; - } - } - return rows; +static void tsdbMayUnTakeMemSnapshot(STsdbReadHandle* pTsdbReadHandle) { +// assert(pTsdbReadHandle != NULL); +// STsdbMemTable* pMemRef = pTsdbReadHandle->pMemRef; +// if (pMemRef == NULL) { // it has been freed +// return; +// } +// +// if (--pMemRef->ref == 0) { +// tsdbUnTakeMemSnapShot(pTsdbReadHandle->pTsdb, &(pMemRef->snapshot)); +// } +// +// pTsdbReadHandle->pMemRef = NULL; } -static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STableGroupInfo* pGroupList, STsdbMeta* pMeta, SArray** psTable) { +//int64_t tsdbGetNumOfRowsInMemTable(tsdbReadHandleT* pHandle) { +// STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; +// +// int64_t rows = 0; +// STsdbMemTable* pMemTable = pTsdbReadHandle->pMemTable; +// if (pMemTable == NULL) { return rows; } +// +//// STableData* pMem = NULL; +//// STableData* pIMem = NULL; +// +//// SMemTable* pMemT = pMemRef->snapshot.mem; +//// SMemTable* pIMemT = pMemRef->snapshot.imem; +// +// size_t size = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); +// for (int32_t i = 0; i < size; ++i) { +// STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); +// +//// if (pMemT && pCheckInfo->tableId < pMemT->maxTables) { +//// pMem = pMemT->tData[pCheckInfo->tableId]; +//// rows += (pMem && pMem->uid == pCheckInfo->tableId) ? pMem->numOfRows : 0; +//// } +//// if (pIMemT && pCheckInfo->tableId < pIMemT->maxTables) { +//// pIMem = pIMemT->tData[pCheckInfo->tableId]; +//// rows += (pIMem && pIMem->uid == pCheckInfo->tableId) ? pIMem->numOfRows : 0; +//// } +// } +// return rows; +//} + +static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo* pGroupList, SArray** psTable) { size_t sizeOfGroup = taosArrayGetSize(pGroupList->pGroupList); - assert(sizeOfGroup >= 1 && pMeta != NULL); + assert(sizeOfGroup >= 1); // allocate buffer in order to load data blocks from file SArray* pTableCheckInfo = taosArrayInit(pGroupList->numOfTables, sizeof(STableCheckInfo)); @@ -288,32 +293,28 @@ static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STa STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(group, j); STableCheckInfo info = { .lastKey = pKeyInfo->lastKey, .pTableObj = pKeyInfo->pTable }; - assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE || - info.pTableObj->type == TSDB_CHILD_TABLE || info.pTableObj->type == TSDB_STREAM_TABLE)); +// assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE || +// info.pTableObj->type == TSDB_CHILD_TABLE || info.pTableObj->type == TSDB_STREAM_TABLE)); - info.tableId.tid = info.pTableObj->tableId.tid; - info.tableId.uid = info.pTableObj->tableId.uid; + info.tableId = pKeyInfo->uid; - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - if (info.lastKey == INT64_MIN || info.lastKey < pQueryHandle->window.skey) { - info.lastKey = pQueryHandle->window.skey; + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { + if (info.lastKey == INT64_MIN || info.lastKey < pTsdbReadHandle->window.skey) { + info.lastKey = pTsdbReadHandle->window.skey; } - assert(info.lastKey >= pQueryHandle->window.skey && info.lastKey <= pQueryHandle->window.ekey); + assert(info.lastKey >= pTsdbReadHandle->window.skey && info.lastKey <= pTsdbReadHandle->window.ekey); } else { - assert(info.lastKey >= pQueryHandle->window.ekey && info.lastKey <= pQueryHandle->window.skey); + assert(info.lastKey >= pTsdbReadHandle->window.ekey && info.lastKey <= pTsdbReadHandle->window.skey); } taosArrayPush(pTableCheckInfo, &info); - tsdbDebug("%p check table uid:%"PRId64", tid:%d from lastKey:%"PRId64" 0x%"PRIx64, pQueryHandle, info.tableId.uid, - info.tableId.tid, info.lastKey, pQueryHandle->qId); + tsdbDebug("%p check table uid:%"PRId64" from lastKey:%"PRId64" 0x%"PRIx64, pTsdbReadHandle, info.tableId, info.lastKey, pTsdbReadHandle->qId); } } - taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar); - +// taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar); size_t gsize = taosArrayGetSize(pTableCheckInfo); - for (int32_t i = 0; i < gsize; ++i) { STableCheckInfo* pInfo = (STableCheckInfo*) taosArrayGet(pTableCheckInfo, i); taosArrayPush(pTable, &pInfo->pTableObj); @@ -323,22 +324,22 @@ static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STa return pTableCheckInfo; } -static void resetCheckInfo(STsdbQueryHandle* pQueryHandle) { - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); +static void resetCheckInfo(STsdbReadHandle* pTsdbReadHandle) { + size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); assert(numOfTables >= 1); // todo apply the lastkey of table check to avoid to load header file for (int32_t i = 0; i < numOfTables; ++i) { - STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i); - pCheckInfo->lastKey = pQueryHandle->window.skey; + STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); + pCheckInfo->lastKey = pTsdbReadHandle->window.skey; pCheckInfo->iter = tSkipListDestroyIter(pCheckInfo->iter); pCheckInfo->iiter = tSkipListDestroyIter(pCheckInfo->iiter); pCheckInfo->initBuf = false; - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - assert(pCheckInfo->lastKey >= pQueryHandle->window.skey); + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { + assert(pCheckInfo->lastKey >= pTsdbReadHandle->window.skey); } else { - assert(pCheckInfo->lastKey <= pQueryHandle->window.skey); + assert(pCheckInfo->lastKey <= pTsdbReadHandle->window.skey); } } } @@ -358,38 +359,38 @@ static SArray* createCheckInfoFromCheckInfo(STableCheckInfo* pCheckInfo, TSKEY s return pNew; } -static bool emptyQueryTimewindow(STsdbQueryHandle* pQueryHandle) { - assert(pQueryHandle != NULL); +static bool emptyQueryTimewindow(STsdbReadHandle* pTsdbReadHandle) { + assert(pTsdbReadHandle != NULL); - STimeWindow* w = &pQueryHandle->window; - bool asc = ASCENDING_TRAVERSE(pQueryHandle->order); + STimeWindow* w = &pTsdbReadHandle->window; + bool asc = ASCENDING_TRAVERSE(pTsdbReadHandle->order); return ((asc && w->skey > w->ekey) || (!asc && w->ekey > w->skey)); } // Update the query time window according to the data time to live(TTL) information, in order to avoid to return // the expired data to client, even it is queried already. -static int64_t getEarliestValidTimestamp(STsdbRepo* pTsdb) { +static int64_t getEarliestValidTimestamp(STsdb* pTsdb) { STsdbCfg* pCfg = &pTsdb->config; int64_t now = taosGetTimestamp(pCfg->precision); return now - (tsTickPerDay[pCfg->precision] * pCfg->keep) + 1; // needs to add one tick } -static void setQueryTimewindow(STsdbQueryHandle* pQueryHandle, STsdbQueryCond* pCond) { - pQueryHandle->window = pCond->twindow; +static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, STsdbQueryCond* pCond) { + pTsdbReadHandle->window = pCond->twindow; bool updateTs = false; - int64_t startTs = getEarliestValidTimestamp(pQueryHandle->pTsdb); - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - if (startTs > pQueryHandle->window.skey) { - pQueryHandle->window.skey = startTs; + int64_t startTs = getEarliestValidTimestamp(pTsdbReadHandle->pTsdb); + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { + if (startTs > pTsdbReadHandle->window.skey) { + pTsdbReadHandle->window.skey = startTs; pCond->twindow.skey = startTs; updateTs = true; } } else { - if (startTs > pQueryHandle->window.ekey) { - pQueryHandle->window.ekey = startTs; + if (startTs > pTsdbReadHandle->window.ekey) { + pTsdbReadHandle->window.ekey = startTs; pCond->twindow.ekey = startTs; updateTs = true; } @@ -397,51 +398,50 @@ static void setQueryTimewindow(STsdbQueryHandle* pQueryHandle, STsdbQueryCond* p if (updateTs) { tsdbDebug("%p update the query time window, old:%" PRId64 " - %" PRId64 ", new:%" PRId64 " - %" PRId64 - ", 0x%" PRIx64, pQueryHandle, pCond->twindow.skey, pCond->twindow.ekey, pQueryHandle->window.skey, - pQueryHandle->window.ekey, pQueryHandle->qId); + ", 0x%" PRIx64, pTsdbReadHandle, pCond->twindow.skey, pCond->twindow.ekey, pTsdbReadHandle->window.skey, + pTsdbReadHandle->window.ekey, pTsdbReadHandle->qId); } } -static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pCond, uint64_t qId, SMemRef* pMemRef) { - STsdbQueryHandle* pQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); - if (pQueryHandle == NULL) { +static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, uint64_t qId, STsdbMemTable* pMemRef) { + STsdbReadHandle* pReadHandle = calloc(1, sizeof(STsdbReadHandle)); + if (pReadHandle == NULL) { goto _end; } - pQueryHandle->order = pCond->order; - pQueryHandle->pTsdb = tsdb; - pQueryHandle->type = TSDB_QUERY_TYPE_ALL; - pQueryHandle->cur.fid = INT32_MIN; - pQueryHandle->cur.win = TSWINDOW_INITIALIZER; - pQueryHandle->checkFiles = true; - pQueryHandle->activeIndex = 0; // current active table index - pQueryHandle->qId = qId; - pQueryHandle->allocSize = 0; - pQueryHandle->locateStart = false; - pQueryHandle->pMemRef = pMemRef; - pQueryHandle->loadType = pCond->type; - - pQueryHandle->outputCapacity = ((STsdbRepo*)tsdb)->config.maxRowsPerFileBlock; - pQueryHandle->loadExternalRow = pCond->loadExternalRows; - pQueryHandle->currentLoadExternalRows = pCond->loadExternalRows; - - if (tsdbInitReadH(&pQueryHandle->rhelper, (STsdbRepo*)tsdb) != 0) { + pReadHandle->order = pCond->order; + pReadHandle->pTsdb = tsdb; + pReadHandle->type = TSDB_QUERY_TYPE_ALL; + pReadHandle->cur.fid = INT32_MIN; + pReadHandle->cur.win = TSWINDOW_INITIALIZER; + pReadHandle->checkFiles = true; + pReadHandle->activeIndex = 0; // current active table index + pReadHandle->qId = qId; + pReadHandle->allocSize = 0; + pReadHandle->locateStart = false; + pReadHandle->loadType = pCond->type; + + pReadHandle->outputCapacity = 4096;//((STsdb*)tsdb)->config.maxRowsPerFileBlock; + pReadHandle->loadExternalRow = pCond->loadExternalRows; + pReadHandle->currentLoadExternalRows = pCond->loadExternalRows; + + if (tsdbInitReadH(&pReadHandle->rhelper, (STsdb*)tsdb) != 0) { goto _end; } - assert(pCond != NULL && pMemRef != NULL); - setQueryTimewindow(pQueryHandle, pCond); + assert(pCond != NULL); + setQueryTimewindow(pReadHandle, pCond); if (pCond->numOfCols > 0) { // allocate buffer in order to load data blocks from file - pQueryHandle->statis = calloc(pCond->numOfCols, sizeof(SDataStatis)); - if (pQueryHandle->statis == NULL) { + pReadHandle->statis = calloc(pCond->numOfCols, sizeof(SDataStatis)); + if (pReadHandle->statis == NULL) { goto _end; } // todo: use list instead of array? - pQueryHandle->pColumns = taosArrayInit(pCond->numOfCols, sizeof(SColumnInfoData)); - if (pQueryHandle->pColumns == NULL) { + pReadHandle->pColumns = taosArrayInit(pCond->numOfCols, sizeof(SColumnInfoData)); + if (pReadHandle->pColumns == NULL) { goto _end; } @@ -449,147 +449,144 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pC SColumnInfoData colInfo = {{0}, 0}; colInfo.info = pCond->colList[i]; - colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCond->colList[i].bytes); + colInfo.pData = calloc(1, EXTRA_BYTES + pReadHandle->outputCapacity * pCond->colList[i].bytes); if (colInfo.pData == NULL) { goto _end; } - taosArrayPush(pQueryHandle->pColumns, &colInfo); - pQueryHandle->statis[i].colId = colInfo.info.colId; + taosArrayPush(pReadHandle->pColumns, &colInfo); + pReadHandle->statis[i].colId = colInfo.info.colId; } - pQueryHandle->defaultLoadColumn = getDefaultLoadColumns(pQueryHandle, true); + pReadHandle->defaultLoadColumn = getDefaultLoadColumns(pReadHandle, true); } - STsdbMeta* pMeta = tsdbGetMeta(tsdb); - assert(pMeta != NULL); +// STsdbMeta* pMeta = NULL;//tsdbGetMeta(tsdb); +// assert(pMeta != NULL); - pQueryHandle->pDataCols = tdNewDataCols(pMeta->maxCols, pQueryHandle->pTsdb->config.maxRowsPerFileBlock); - if (pQueryHandle->pDataCols == NULL) { - tsdbError("%p failed to malloc buf for pDataCols, %"PRIu64, pQueryHandle, pQueryHandle->qId); + pReadHandle->pDataCols = tdNewDataCols(1000, pReadHandle->pTsdb->config.maxRowsPerFileBlock); + if (pReadHandle->pDataCols == NULL) { + tsdbError("%p failed to malloc buf for pDataCols, %"PRIu64, pReadHandle, pReadHandle->qId); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _end; } - tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); - tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); + tsdbInitDataBlockLoadInfo(&pReadHandle->dataBlockLoadInfo); + tsdbInitCompBlockLoadInfo(&pReadHandle->compBlockLoadInfo); - return (TsdbQueryHandleT) pQueryHandle; + return (tsdbReadHandleT)pReadHandle; _end: - tsdbCleanupQueryHandle(pQueryHandle); +// tsdbCleanupQueryHandle(pTsdbReadHandle); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; } -TsdbQueryHandleT* tsdbQueryTables(STsdbRepo* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId, SMemRef* pRef) { - STsdbQueryHandle* pQueryHandle = tsdbQueryTablesImpl(tsdb, pCond, qId, pRef); - if (pQueryHandle == NULL) { +tsdbReadHandleT* tsdbQueryTables(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId, void* pRef) { + STsdbReadHandle* pTsdbReadHandle = tsdbQueryTablesImpl(tsdb, pCond, qId, pRef); + if (pTsdbReadHandle == NULL) { return NULL; } - if (emptyQueryTimewindow(pQueryHandle)) { - return (TsdbQueryHandleT*) pQueryHandle; + if (emptyQueryTimewindow(pTsdbReadHandle)) { + return (tsdbReadHandleT*) pTsdbReadHandle; } - STsdbMeta* pMeta = tsdbGetMeta(tsdb); - assert(pMeta != NULL); - SArray* psTable = NULL; // todo apply the lastkey of table check to avoid to load header file - pQueryHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pQueryHandle, groupList, pMeta, &psTable); - if (pQueryHandle->pTableCheckInfo == NULL) { - tsdbCleanupQueryHandle(pQueryHandle); + pTsdbReadHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pTsdbReadHandle, groupList, &psTable); + if (pTsdbReadHandle->pTableCheckInfo == NULL) { +// tsdbCleanupQueryHandle(pTsdbReadHandle); taosArrayDestroy(psTable); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; } - tsdbMayTakeMemSnapshot(pQueryHandle, psTable); +// tsdbMayTakeMemSnapshot(pTsdbReadHandle, psTable); - tsdbDebug("%p total numOfTable:%" PRIzu " in query, 0x%"PRIx64, pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo), pQueryHandle->qId); - return (TsdbQueryHandleT) pQueryHandle; + tsdbDebug("%p total numOfTable:%" PRIzu " in query, 0x%"PRIx64, pTsdbReadHandle, taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo), pTsdbReadHandle->qId); + return (tsdbReadHandleT) pTsdbReadHandle; } -void tsdbResetQueryHandle(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond) { - STsdbQueryHandle* pQueryHandle = queryHandle; +void tsdbResetQueryHandle(tsdbReadHandleT queryHandle, STsdbQueryCond *pCond) { + STsdbReadHandle* pTsdbReadHandle = queryHandle; - if (emptyQueryTimewindow(pQueryHandle)) { - if (pCond->order != pQueryHandle->order) { - pQueryHandle->order = pCond->order; - SWAP(pQueryHandle->window.skey, pQueryHandle->window.ekey, int64_t); + if (emptyQueryTimewindow(pTsdbReadHandle)) { + if (pCond->order != pTsdbReadHandle->order) { + pTsdbReadHandle->order = pCond->order; + SWAP(pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, int64_t); } return; } - pQueryHandle->order = pCond->order; - pQueryHandle->window = pCond->twindow; - pQueryHandle->type = TSDB_QUERY_TYPE_ALL; - pQueryHandle->cur.fid = -1; - pQueryHandle->cur.win = TSWINDOW_INITIALIZER; - pQueryHandle->checkFiles = true; - pQueryHandle->activeIndex = 0; // current active table index - pQueryHandle->locateStart = false; - pQueryHandle->loadExternalRow = pCond->loadExternalRows; + pTsdbReadHandle->order = pCond->order; + pTsdbReadHandle->window = pCond->twindow; + pTsdbReadHandle->type = TSDB_QUERY_TYPE_ALL; + pTsdbReadHandle->cur.fid = -1; + pTsdbReadHandle->cur.win = TSWINDOW_INITIALIZER; + pTsdbReadHandle->checkFiles = true; + pTsdbReadHandle->activeIndex = 0; // current active table index + pTsdbReadHandle->locateStart = false; + pTsdbReadHandle->loadExternalRow = pCond->loadExternalRows; if (ASCENDING_TRAVERSE(pCond->order)) { - assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey); + assert(pTsdbReadHandle->window.skey <= pTsdbReadHandle->window.ekey); } else { - assert(pQueryHandle->window.skey >= pQueryHandle->window.ekey); + assert(pTsdbReadHandle->window.skey >= pTsdbReadHandle->window.ekey); } // allocate buffer in order to load data blocks from file - memset(pQueryHandle->statis, 0, sizeof(SDataStatis)); + memset(pTsdbReadHandle->statis, 0, sizeof(SDataStatis)); - tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); - tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); + tsdbInitDataBlockLoadInfo(&pTsdbReadHandle->dataBlockLoadInfo); + tsdbInitCompBlockLoadInfo(&pTsdbReadHandle->compBlockLoadInfo); - resetCheckInfo(pQueryHandle); + resetCheckInfo(pTsdbReadHandle); } -void tsdbResetQueryHandleForNewTable(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond, STableGroupInfo* groupList) { - STsdbQueryHandle* pQueryHandle = queryHandle; +void tsdbResetQueryHandleForNewTable(tsdbReadHandleT queryHandle, STsdbQueryCond *pCond, STableGroupInfo* groupList) { + STsdbReadHandle* pTsdbReadHandle = queryHandle; - pQueryHandle->order = pCond->order; - pQueryHandle->window = pCond->twindow; - pQueryHandle->type = TSDB_QUERY_TYPE_ALL; - pQueryHandle->cur.fid = -1; - pQueryHandle->cur.win = TSWINDOW_INITIALIZER; - pQueryHandle->checkFiles = true; - pQueryHandle->activeIndex = 0; // current active table index - pQueryHandle->locateStart = false; - pQueryHandle->loadExternalRow = pCond->loadExternalRows; + pTsdbReadHandle->order = pCond->order; + pTsdbReadHandle->window = pCond->twindow; + pTsdbReadHandle->type = TSDB_QUERY_TYPE_ALL; + pTsdbReadHandle->cur.fid = -1; + pTsdbReadHandle->cur.win = TSWINDOW_INITIALIZER; + pTsdbReadHandle->checkFiles = true; + pTsdbReadHandle->activeIndex = 0; // current active table index + pTsdbReadHandle->locateStart = false; + pTsdbReadHandle->loadExternalRow = pCond->loadExternalRows; if (ASCENDING_TRAVERSE(pCond->order)) { - assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey); + assert(pTsdbReadHandle->window.skey <= pTsdbReadHandle->window.ekey); } else { - assert(pQueryHandle->window.skey >= pQueryHandle->window.ekey); + assert(pTsdbReadHandle->window.skey >= pTsdbReadHandle->window.ekey); } // allocate buffer in order to load data blocks from file - memset(pQueryHandle->statis, 0, sizeof(SDataStatis)); + memset(pTsdbReadHandle->statis, 0, sizeof(SDataStatis)); - tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); - tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); + tsdbInitDataBlockLoadInfo(&pTsdbReadHandle->dataBlockLoadInfo); + tsdbInitCompBlockLoadInfo(&pTsdbReadHandle->compBlockLoadInfo); SArray* pTable = NULL; - STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb); +// STsdbMeta* pMeta = tsdbGetMeta(pTsdbReadHandle->pTsdb); - pQueryHandle->pTableCheckInfo = destroyTableCheckInfo(pQueryHandle->pTableCheckInfo); +// pTsdbReadHandle->pTableCheckInfo = destroyTableCheckInfo(pTsdbReadHandle->pTableCheckInfo); - pQueryHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pQueryHandle, groupList, pMeta, &pTable); - if (pQueryHandle->pTableCheckInfo == NULL) { - tsdbCleanupQueryHandle(pQueryHandle); + pTsdbReadHandle->pTableCheckInfo = NULL;//createCheckInfoFromTableGroup(pTsdbReadHandle, groupList, pMeta, &pTable); + if (pTsdbReadHandle->pTableCheckInfo == NULL) { +// tsdbCleanupQueryHandle(pTsdbReadHandle); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; } - pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); - pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); +// pTsdbReadHandle->prev = doFreeColumnInfoData(pTsdbReadHandle->prev); +// pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next); } -TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pMemRef) { +tsdbReadHandleT tsdbQueryLastRow(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) { pCond->twindow = updateLastrowForEachGroup(groupList); // no qualified table @@ -597,56 +594,56 @@ TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STable return NULL; } - STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); - if (pQueryHandle == NULL) { + STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); + if (pTsdbReadHandle == NULL) { return NULL; } - int32_t code = checkForCachedLastRow(pQueryHandle, groupList); + int32_t code = checkForCachedLastRow(pTsdbReadHandle, groupList); if (code != TSDB_CODE_SUCCESS) { // set the numOfTables to be 0 terrno = code; return NULL; } assert(pCond->order == TSDB_ORDER_ASC && pCond->twindow.skey <= pCond->twindow.ekey); - if (pQueryHandle->cachelastrow) { - pQueryHandle->type = TSDB_QUERY_TYPE_LAST; + if (pTsdbReadHandle->cachelastrow) { + pTsdbReadHandle->type = TSDB_QUERY_TYPE_LAST; } - return pQueryHandle; + return pTsdbReadHandle; } - -TsdbQueryHandleT tsdbQueryCacheLast(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pMemRef) { - STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); - if (pQueryHandle == NULL) { +#if 0 +tsdbReadHandleT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) { + STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); + if (pTsdbReadHandle == NULL) { return NULL; } - int32_t code = checkForCachedLast(pQueryHandle); + int32_t code = checkForCachedLast(pTsdbReadHandle); if (code != TSDB_CODE_SUCCESS) { // set the numOfTables to be 0 terrno = code; return NULL; } - if (pQueryHandle->cachelastrow) { - pQueryHandle->type = TSDB_QUERY_TYPE_LAST; + if (pTsdbReadHandle->cachelastrow) { + pTsdbReadHandle->type = TSDB_QUERY_TYPE_LAST; } - return pQueryHandle; + return pTsdbReadHandle; } - -SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle) { +#endif +SArray* tsdbGetQueriedTableList(tsdbReadHandleT *pHandle) { assert(pHandle != NULL); - STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) pHandle; + STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) pHandle; - size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + size_t size = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); SArray* res = taosArrayInit(size, POINTER_BYTES); for(int32_t i = 0; i < size; ++i) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); taosArrayPush(res, &pCheckInfo->pTableObj); } @@ -668,11 +665,11 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr SArray* px = taosArrayInit(4, sizeof(STableKeyInfo)); for (int32_t j = 0; j < numOfTables; ++j) { STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(oneGroup, j); - if (window->skey <= pInfo->lastKey && ((STable*)pInfo->pTable)->lastKey != TSKEY_INITIAL_VAL) { - taosArrayPush(px, pInfo); - pNew->numOfTables += 1; - break; - } +// if (window->skey <= pInfo->lastKey && ((STable*)pInfo->pTable)->lastKey != TSKEY_INITIAL_VAL) { +// taosArrayPush(px, pInfo); +// pNew->numOfTables += 1; +// break; +// } } // there are no data in this group @@ -686,7 +683,7 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr return pNew; } -TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pRef) { +tsdbReadHandleT tsdbQueryRowsInExternalWindow(STsdb *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pRef) { STableGroupInfo* pNew = trimTableGroup(&pCond->twindow, groupList); if (pNew->numOfTables == 0) { @@ -701,17 +698,14 @@ TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond* } } - STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, pNew, qId, pRef); - pQueryHandle->loadExternalRow = true; - pQueryHandle->currentLoadExternalRows = true; + STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, pNew, qId, pRef); + pTsdbReadHandle->loadExternalRow = true; + pTsdbReadHandle->currentLoadExternalRows = true; - return pQueryHandle; + return pTsdbReadHandle; } -static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCheckInfo) { - STable* pTable = pCheckInfo->pTableObj; - assert(pTable != NULL); - +static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pCheckInfo) { if (pCheckInfo->initBuf) { return true; } @@ -720,33 +714,29 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh int32_t order = pHandle->order; // no data in buffer, abort - if (pHandle->pMemRef->snapshot.mem == NULL && pHandle->pMemRef->snapshot.imem == NULL) { - return false; - } - - assert(pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL); - - STableData* pMem = NULL; - STableData* pIMem = NULL; - - SMemTable* pMemT = pHandle->pMemRef->snapshot.mem; - SMemTable* pIMemT = pHandle->pMemRef->snapshot.imem; - - if (pMemT && pCheckInfo->tableId.tid < pMemT->maxTables) { - pMem = pMemT->tData[pCheckInfo->tableId.tid]; - if (pMem != NULL && pMem->uid == pCheckInfo->tableId.uid) { // check uid - TKEY tLastKey = keyToTkey(pCheckInfo->lastKey); +// if (pHandle->pMemTable->snapshot.mem == NULL && pHandle->pMemTable->snapshot.imem == NULL) { +// return false; +// } +// +// assert(pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL); +// + STbData** pMem = NULL; + STbData** pIMem = NULL; + + TKEY tLastKey = 0; /// keyToTkey(pCheckInfo->lastKey); + if (pHandle->pTsdb->mem != NULL) { + pMem = taosHashGet(pHandle->pTsdb->mem->pHashIdx, &pCheckInfo->tableId, sizeof(pCheckInfo->tableId)); + if (pMem != NULL) { pCheckInfo->iter = - tSkipListCreateIterFromVal(pMem->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order); + tSkipListCreateIterFromVal((*pMem)->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order); } } - if (pIMemT && pCheckInfo->tableId.tid < pIMemT->maxTables) { - pIMem = pIMemT->tData[pCheckInfo->tableId.tid]; - if (pIMem != NULL && pIMem->uid == pCheckInfo->tableId.uid) { // check uid - TKEY tLastKey = keyToTkey(pCheckInfo->lastKey); + if (pHandle->pTsdb->imem != NULL) { + pIMem = taosHashGet(pHandle->pTsdb->imem->pHashIdx, &pCheckInfo->tableId, sizeof(pCheckInfo->tableId)); + if (pIMem != NULL) { pCheckInfo->iiter = - tSkipListCreateIterFromVal(pIMem->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order); + tSkipListCreateIterFromVal((*pIMem)->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order); } } @@ -767,10 +757,9 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SMemRow row = (SMemRow)SL_GET_NODE_DATA(node); TSKEY key = memRowKey(row); // first timestamp in buffer - tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 + tsdbDebug("%p uid:%" PRId64 ", check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", 0x%"PRIx64, - pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pMem->keyFirst, pMem->keyLast, - pCheckInfo->lastKey, pMem->numOfRows, pHandle->qId); + pHandle, pCheckInfo->tableId, key, order, (*pMem)->keyMin, (*pMem)->keyMax, pCheckInfo->lastKey, (*pMem)->nrows, pHandle->qId); if (ASCENDING_TRAVERSE(order)) { assert(pCheckInfo->lastKey <= key); @@ -779,8 +768,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh } } else { - tsdbDebug("%p uid:%"PRId64", tid:%d no data in mem, 0x%"PRIx64, pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, - pHandle->qId); + tsdbDebug("%p uid:%"PRId64", no data in mem, 0x%"PRIx64, pHandle, pCheckInfo->tableId, pHandle->qId); } if (!imemEmpty) { @@ -789,10 +777,9 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SMemRow row = (SMemRow)SL_GET_NODE_DATA(node); TSKEY key = memRowKey(row); // first timestamp in buffer - tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 + tsdbDebug("%p uid:%" PRId64 ", check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", 0x%"PRIx64, - pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pIMem->keyFirst, pIMem->keyLast, - pCheckInfo->lastKey, pIMem->numOfRows, pHandle->qId); + pHandle, pCheckInfo->tableId, key, order, (*pIMem)->keyMin, (*pIMem)->keyMax, pCheckInfo->lastKey, (*pIMem)->nrows, pHandle->qId); if (ASCENDING_TRAVERSE(order)) { assert(pCheckInfo->lastKey <= key); @@ -800,7 +787,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh assert(pCheckInfo->lastKey >= key); } } else { - tsdbDebug("%p uid:%"PRId64", tid:%d no data in imem, 0x%"PRIx64, pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, + tsdbDebug("%p uid:%"PRId64", no data in imem, 0x%"PRIx64, pHandle, pCheckInfo->tableId, pHandle->qId); } @@ -973,17 +960,13 @@ static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) { return hasNext; } -static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { +static bool hasMoreDataInCache(STsdbReadHandle* pHandle) { STsdbCfg *pCfg = &pHandle->pTsdb->config; size_t size = taosArrayGetSize(pHandle->pTableCheckInfo); assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1); pHandle->cur.fid = INT32_MIN; STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - - STable* pTable = pCheckInfo->pTableObj; - assert(pTable != NULL); - if (!pCheckInfo->initBuf) { initTableMemIterator(pHandle, pCheckInfo); } @@ -994,8 +977,8 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { } pCheckInfo->lastKey = memRowKey(row); // first timestamp in buffer - tsdbDebug("%p uid:%" PRId64", tid:%d check data in buffer from skey:%" PRId64 ", order:%d, 0x%"PRIx64, pHandle, - pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pCheckInfo->lastKey, pHandle->order, pHandle->qId); + tsdbDebug("%p uid:%" PRId64", check data in buffer from skey:%" PRId64 ", order:%d, 0x%"PRIx64, pHandle, + pCheckInfo->tableId, pCheckInfo->lastKey, pHandle->order, pHandle->qId); // all data in mem are checked already. if ((pCheckInfo->lastKey > pHandle->window.ekey && ASCENDING_TRAVERSE(pHandle->order)) || @@ -1068,21 +1051,21 @@ static int32_t binarySearchForBlock(SBlock* pBlock, int32_t numOfBlocks, TSKEY s return midSlot; } -static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t index, int32_t* numOfBlocks) { +static int32_t loadBlockInfo(STsdbReadHandle * pTsdbReadHandle, int32_t index, int32_t* numOfBlocks) { int32_t code = 0; - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, index); + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, index); pCheckInfo->numOfBlocks = 0; - if (tsdbSetReadTable(&pQueryHandle->rhelper, pCheckInfo->pTableObj) != TSDB_CODE_SUCCESS) { + if (tsdbSetReadTable(&pTsdbReadHandle->rhelper, pCheckInfo->pTableObj) != TSDB_CODE_SUCCESS) { code = terrno; return code; } - SBlockIdx* compIndex = pQueryHandle->rhelper.pBlkIdx; + SBlockIdx* compIndex = pTsdbReadHandle->rhelper.pBlkIdx; // no data block in this file, try next file - if (compIndex == NULL || compIndex->uid != pCheckInfo->tableId.uid) { + if (compIndex == NULL || compIndex->uid != pCheckInfo->tableId) { return 0; // no data blocks in the file belongs to pCheckInfo->pTable } @@ -1100,21 +1083,21 @@ static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t index, int pCheckInfo->compSize = compIndex->len; } - if (tsdbLoadBlockInfo(&(pQueryHandle->rhelper), (void*)(pCheckInfo->pCompInfo)) < 0) { + if (tsdbLoadBlockInfo(&(pTsdbReadHandle->rhelper), (void*)(pCheckInfo->pCompInfo)) < 0) { return terrno; } SBlockInfo* pCompInfo = pCheckInfo->pCompInfo; TSKEY s = TSKEY_INITIAL_VAL, e = TSKEY_INITIAL_VAL; - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - assert(pCheckInfo->lastKey <= pQueryHandle->window.ekey && pQueryHandle->window.skey <= pQueryHandle->window.ekey); + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { + assert(pCheckInfo->lastKey <= pTsdbReadHandle->window.ekey && pTsdbReadHandle->window.skey <= pTsdbReadHandle->window.ekey); } else { - assert(pCheckInfo->lastKey >= pQueryHandle->window.ekey && pQueryHandle->window.skey >= pQueryHandle->window.ekey); + assert(pCheckInfo->lastKey >= pTsdbReadHandle->window.ekey && pTsdbReadHandle->window.skey >= pTsdbReadHandle->window.ekey); } - s = MIN(pCheckInfo->lastKey, pQueryHandle->window.ekey); - e = MAX(pCheckInfo->lastKey, pQueryHandle->window.ekey); + s = MIN(pCheckInfo->lastKey, pTsdbReadHandle->window.ekey); + e = MAX(pCheckInfo->lastKey, pTsdbReadHandle->window.ekey); // discard the unqualified data block based on the query time window int32_t start = binarySearchForBlock(pCompInfo->blocks, compIndex->numOfBlocks, s, TSDB_ORDER_ASC); @@ -1139,26 +1122,26 @@ static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t index, int return 0; } -static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlocks) { +static int32_t getFileCompInfo(STsdbReadHandle* pTsdbReadHandle, int32_t* numOfBlocks) { // load all the comp offset value for all tables in this file int32_t code = TSDB_CODE_SUCCESS; *numOfBlocks = 0; - pQueryHandle->cost.headFileLoad += 1; + pTsdbReadHandle->cost.headFileLoad += 1; int64_t s = taosGetTimestampUs(); size_t numOfTables = 0; - if (pQueryHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { - code = loadBlockInfo(pQueryHandle, pQueryHandle->activeIndex, numOfBlocks); - } else if (pQueryHandle->loadType == BLOCK_LOAD_OFFSET_SEQ_ORDER) { - numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + if (pTsdbReadHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { + code = loadBlockInfo(pTsdbReadHandle, pTsdbReadHandle->activeIndex, numOfBlocks); + } else if (pTsdbReadHandle->loadType == BLOCK_LOAD_OFFSET_SEQ_ORDER) { + numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); for (int32_t i = 0; i < numOfTables; ++i) { - code = loadBlockInfo(pQueryHandle, i, numOfBlocks); + code = loadBlockInfo(pTsdbReadHandle, i, numOfBlocks); if (code != TSDB_CODE_SUCCESS) { int64_t e = taosGetTimestampUs(); - pQueryHandle->cost.headFileLoadTime += (e - s); + pTsdbReadHandle->cost.headFileLoadTime += (e - s); return code; } } @@ -1167,57 +1150,57 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo } int64_t e = taosGetTimestampUs(); - pQueryHandle->cost.headFileLoadTime += (e - s); + pTsdbReadHandle->cost.headFileLoadTime += (e - s); return code; } -static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) { +static int32_t doLoadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) { int64_t st = taosGetTimestampUs(); - STSchema *pSchema = tsdbGetTableSchema(pCheckInfo->pTableObj); - int32_t code = tdInitDataCols(pQueryHandle->pDataCols, pSchema); + STSchema *pSchema = NULL;//tsdbGetTableSchema(pCheckInfo->pTableObj); + int32_t code = tdInitDataCols(pTsdbReadHandle->pDataCols, pSchema); if (code != TSDB_CODE_SUCCESS) { - tsdbError("%p failed to malloc buf for pDataCols, 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); + tsdbError("%p failed to malloc buf for pDataCols, 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _error; } - code = tdInitDataCols(pQueryHandle->rhelper.pDCols[0], pSchema); + code = tdInitDataCols(pTsdbReadHandle->rhelper.pDCols[0], pSchema); if (code != TSDB_CODE_SUCCESS) { - tsdbError("%p failed to malloc buf for rhelper.pDataCols[0], 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); + tsdbError("%p failed to malloc buf for rhelper.pDataCols[0], 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _error; } - code = tdInitDataCols(pQueryHandle->rhelper.pDCols[1], pSchema); + code = tdInitDataCols(pTsdbReadHandle->rhelper.pDCols[1], pSchema); if (code != TSDB_CODE_SUCCESS) { - tsdbError("%p failed to malloc buf for rhelper.pDataCols[1], 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); + tsdbError("%p failed to malloc buf for rhelper.pDataCols[1], 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _error; } - int16_t* colIds = pQueryHandle->defaultLoadColumn->pData; + int16_t* colIds = pTsdbReadHandle->defaultLoadColumn->pData; - int32_t ret = tsdbLoadBlockDataCols(&(pQueryHandle->rhelper), pBlock, pCheckInfo->pCompInfo, colIds, (int)(QH_GET_NUM_OF_COLS(pQueryHandle))); + int32_t ret = tsdbLoadBlockDataCols(&(pTsdbReadHandle->rhelper), pBlock, pCheckInfo->pCompInfo, colIds, (int)(QH_GET_NUM_OF_COLS(pTsdbReadHandle))); if (ret != TSDB_CODE_SUCCESS) { int32_t c = terrno; assert(c != TSDB_CODE_SUCCESS); goto _error; } - SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo; + SDataBlockLoadInfo* pBlockLoadInfo = &pTsdbReadHandle->dataBlockLoadInfo; - pBlockLoadInfo->fileGroup = pQueryHandle->pFileGroup; - pBlockLoadInfo->slot = pQueryHandle->cur.slot; - pBlockLoadInfo->tid = pCheckInfo->pTableObj->tableId.tid; + pBlockLoadInfo->fileGroup = pTsdbReadHandle->pFileGroup; + pBlockLoadInfo->slot = pTsdbReadHandle->cur.slot; + pBlockLoadInfo->uid = pCheckInfo->pTableObj->uid; - SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; + SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; assert(pCols->numOfRows != 0 && pCols->numOfRows <= pBlock->numOfRows); pBlock->numOfRows = pCols->numOfRows; // Convert from TKEY to TSKEY for primary timestamp column if current block has timestamp before 1970-01-01T00:00:00Z - if(pBlock->keyFirst < 0 && colIds[0] == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + if(pBlock->keyFirst < 0 && colIds[0] == PRIMARYKEY_TIMESTAMP_COL_ID) { int64_t* src = pCols->cols[0].pData; for(int32_t i = 0; i < pBlock->numOfRows; ++i) { src[i] = tdGetKey(src[i]); @@ -1225,60 +1208,60 @@ static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBloc } int64_t elapsedTime = (taosGetTimestampUs() - st); - pQueryHandle->cost.blockLoadTime += elapsedTime; + pTsdbReadHandle->cost.blockLoadTime += elapsedTime; tsdbDebug("%p load file block into buffer, index:%d, brange:%"PRId64"-%"PRId64", rows:%d, elapsed time:%"PRId64 " us, 0x%"PRIx64, - pQueryHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, elapsedTime, pQueryHandle->qId); + pTsdbReadHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, elapsedTime, pTsdbReadHandle->qId); return TSDB_CODE_SUCCESS; _error: pBlock->numOfRows = 0; tsdbError("%p error occurs in loading file block, index:%d, brange:%"PRId64"-%"PRId64", rows:%d, 0x%"PRIx64, - pQueryHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, pQueryHandle->qId); + pTsdbReadHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, pTsdbReadHandle->qId); return terrno; } -static int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBlockInfo); -static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end); -static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, int32_t numOfCols); -static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle); -static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos); +static int32_t getEndPosInDataBlock(STsdbReadHandle* pTsdbReadHandle, SDataBlockInfo* pBlockInfo); +static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end); +static void moveDataToFront(STsdbReadHandle* pTsdbReadHandle, int32_t numOfRows, int32_t numOfCols); +static void doCheckGeneratedBlockRange(STsdbReadHandle* pTsdbReadHandle); +static void copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos); -static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo){ - SQueryFilePos* cur = &pQueryHandle->cur; - STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; +static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo){ + SQueryFilePos* cur = &pTsdbReadHandle->cur; + STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); TSKEY key; int32_t code = TSDB_CODE_SUCCESS; - /*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo); + /*bool hasData = */ initTableMemIterator(pTsdbReadHandle, pCheckInfo); assert(cur->pos >= 0 && cur->pos <= binfo.rows); - key = extractFirstTraverseKey(pCheckInfo, pQueryHandle->order, pCfg->update); + key = extractFirstTraverseKey(pCheckInfo, pTsdbReadHandle->order, pCfg->update); if (key != TSKEY_INITIAL_VAL) { - tsdbDebug("%p key in mem:%"PRId64", 0x%"PRIx64, pQueryHandle, key, pQueryHandle->qId); + tsdbDebug("%p key in mem:%"PRId64", 0x%"PRIx64, pTsdbReadHandle, key, pTsdbReadHandle->qId); } else { - tsdbDebug("%p no data in mem, 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); + tsdbDebug("%p no data in mem, 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId); } - if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) || - (!ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) { + if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) || + (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) { - if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) || - (!ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) { + if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) || + (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) { // do not load file block into buffer - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1; + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1; - TSKEY maxKey = ASCENDING_TRAVERSE(pQueryHandle->order)? (binfo.window.skey - step):(binfo.window.ekey - step); - cur->rows = tsdbReadRowsFromCache(pCheckInfo, maxKey, pQueryHandle->outputCapacity, &cur->win, pQueryHandle); - pQueryHandle->realNumOfRows = cur->rows; + TSKEY maxKey = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? (binfo.window.skey - step):(binfo.window.ekey - step); + cur->rows = tsdbReadRowsFromCache(pCheckInfo, maxKey, pTsdbReadHandle->outputCapacity, &cur->win, pTsdbReadHandle); + pTsdbReadHandle->realNumOfRows = cur->rows; // update the last key value pCheckInfo->lastKey = cur->win.ekey + step; - if (!ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { SWAP(cur->win.skey, cur->win.ekey, TSKEY); } @@ -1289,11 +1272,11 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p // return error, add test cases - if ((code = doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { + if ((code = doLoadFileDataBlock(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { return code; } - doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); + doMergeTwoLevelData(pTsdbReadHandle, pCheckInfo, pBlock); } else { /* * no data in cache, only load data from file @@ -1301,19 +1284,19 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p * * Here the buffer is not enough, so only part of file block can be loaded into memory buffer */ - assert(pQueryHandle->outputCapacity >= binfo.rows); - int32_t endPos = getEndPosInDataBlock(pQueryHandle, &binfo); + assert(pTsdbReadHandle->outputCapacity >= binfo.rows); + int32_t endPos = getEndPosInDataBlock(pTsdbReadHandle, &binfo); - if ((cur->pos == 0 && endPos == binfo.rows -1 && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (cur->pos == (binfo.rows - 1) && endPos == 0 && (!ASCENDING_TRAVERSE(pQueryHandle->order)))) { - pQueryHandle->realNumOfRows = binfo.rows; + if ((cur->pos == 0 && endPos == binfo.rows -1 && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + (cur->pos == (binfo.rows - 1) && endPos == 0 && (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)))) { + pTsdbReadHandle->realNumOfRows = binfo.rows; cur->rows = binfo.rows; cur->win = binfo.window; cur->mixBlock = false; cur->blockCompleted = true; - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { cur->lastKey = binfo.window.ekey + 1; cur->pos = binfo.rows; } else { @@ -1321,17 +1304,17 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p cur->pos = -1; } } else { // partially copy to dest buffer - copyAllRemainRowsFromFileBlock(pQueryHandle, pCheckInfo, &binfo, endPos); + copyAllRemainRowsFromFileBlock(pTsdbReadHandle, pCheckInfo, &binfo, endPos); cur->mixBlock = true; } assert(cur->blockCompleted); if (cur->rows == binfo.rows) { tsdbDebug("%p whole file block qualified, brange:%"PRId64"-%"PRId64", rows:%d, lastKey:%"PRId64", %"PRIx64, - pQueryHandle, cur->win.skey, cur->win.ekey, cur->rows, cur->lastKey, pQueryHandle->qId); + pTsdbReadHandle, cur->win.skey, cur->win.ekey, cur->rows, cur->lastKey, pTsdbReadHandle->qId); } else { tsdbDebug("%p create data block from remain file block, brange:%"PRId64"-%"PRId64", rows:%d, total:%d, lastKey:%"PRId64", %"PRIx64, - pQueryHandle, cur->win.skey, cur->win.ekey, cur->rows, binfo.rows, cur->lastKey, pQueryHandle->qId); + pTsdbReadHandle, cur->win.skey, cur->win.ekey, cur->rows, binfo.rows, cur->lastKey, pTsdbReadHandle->qId); } } @@ -1339,58 +1322,58 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p return code; } -static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, bool* exists) { - SQueryFilePos* cur = &pQueryHandle->cur; +static int32_t loadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, bool* exists) { + SQueryFilePos* cur = &pTsdbReadHandle->cur; int32_t code = TSDB_CODE_SUCCESS; - bool asc = ASCENDING_TRAVERSE(pQueryHandle->order); + bool asc = ASCENDING_TRAVERSE(pTsdbReadHandle->order); if (asc) { // query ended in/started from current block - if (pQueryHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) { - if ((code = doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { + if (pTsdbReadHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) { + if ((code = doLoadFileDataBlock(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { *exists = false; return code; } - SDataCols* pTSCol = pQueryHandle->rhelper.pDCols[0]; + SDataCols* pTSCol = pTsdbReadHandle->rhelper.pDCols[0]; assert(pTSCol->cols->type == TSDB_DATA_TYPE_TIMESTAMP && pTSCol->numOfRows == pBlock->numOfRows); if (pCheckInfo->lastKey > pBlock->keyFirst) { cur->pos = - binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order); + binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pTsdbReadHandle->order); } else { cur->pos = 0; } assert(pCheckInfo->lastKey <= pBlock->keyLast); - doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); + doMergeTwoLevelData(pTsdbReadHandle, pCheckInfo, pBlock); } else { // the whole block is loaded in to buffer cur->pos = asc? 0:(pBlock->numOfRows - 1); - code = handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo); + code = handleDataMergeIfNeeded(pTsdbReadHandle, pBlock, pCheckInfo); } } else { //desc order, query ended in current block - if (pQueryHandle->window.ekey > pBlock->keyFirst || pCheckInfo->lastKey < pBlock->keyLast) { - if ((code = doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { + if (pTsdbReadHandle->window.ekey > pBlock->keyFirst || pCheckInfo->lastKey < pBlock->keyLast) { + if ((code = doLoadFileDataBlock(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { *exists = false; return code; } - SDataCols* pTsCol = pQueryHandle->rhelper.pDCols[0]; + SDataCols* pTsCol = pTsdbReadHandle->rhelper.pDCols[0]; if (pCheckInfo->lastKey < pBlock->keyLast) { - cur->pos = binarySearchForKey(pTsCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order); + cur->pos = binarySearchForKey(pTsCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pTsdbReadHandle->order); } else { cur->pos = pBlock->numOfRows - 1; } assert(pCheckInfo->lastKey >= pBlock->keyFirst); - doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); + doMergeTwoLevelData(pTsdbReadHandle, pCheckInfo, pBlock); } else { cur->pos = asc? 0:(pBlock->numOfRows-1); - code = handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo); + code = handleDataMergeIfNeeded(pTsdbReadHandle, pBlock, pCheckInfo); } } - *exists = pQueryHandle->realNumOfRows > 0; + *exists = pTsdbReadHandle->realNumOfRows > 0; return code; } @@ -1456,11 +1439,11 @@ static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) { return midPos; } -static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end) { +static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end) { char* pData = NULL; - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1 : -1; - SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; + SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; TSKEY* tsArray = pCols->cols[0].pData; int32_t num = end - start + 1; @@ -1470,12 +1453,12 @@ static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t c return numOfRows; } - int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns); + int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns); //data in buffer has greater timestamp, copy data in file block int32_t i = 0, j = 0; while(i < requiredNumOfCols && j < pCols->numOfCols) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); SDataCol* src = &pCols->cols[j]; if (src->colId < pColInfo->info.colId) { @@ -1485,7 +1468,7 @@ static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t c int32_t bytes = pColInfo->info.bytes; - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; } else { pData = (char*)pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes; @@ -1523,8 +1506,8 @@ static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t c } while (i < requiredNumOfCols) { // the remain columns are all null data - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; } else { pData = (char*)pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes; @@ -1544,15 +1527,15 @@ static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t c i++; } - pQueryHandle->cur.win.ekey = tsArray[end]; - pQueryHandle->cur.lastKey = tsArray[end] + step; + pTsdbReadHandle->cur.win.ekey = tsArray[end]; + pTsdbReadHandle->cur.lastKey = tsArray[end] + step; return numOfRows + num; } // Note: row1 always has high priority -static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, - SMemRow row1, SMemRow row2, int32_t numOfCols, STable* pTable, +static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, + SMemRow row1, SMemRow row2, int32_t numOfCols, uint64_t uid, STSchema* pSchema1, STSchema* pSchema2, bool forceSetNull) { char* pData = NULL; STSchema* pSchema; @@ -1570,8 +1553,9 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfColsOfRow1 = 0; if (pSchema1 == NULL) { - pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); + pSchema1 = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, uid, 0); } + if(isRow1DataRow) { numOfColsOfRow1 = schemaNCols(pSchema1); } else { @@ -1582,7 +1566,7 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, if(row2) { isRow2DataRow = isDataRow(row2); if (pSchema2 == NULL) { - pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); + pSchema2 = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, uid, 0); } if(isRow2DataRow) { numOfColsOfRow2 = schemaNCols(pSchema2); @@ -1594,9 +1578,9 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t i = 0, j = 0, k = 0; while(i < numOfCols && (j < numOfColsOfRow1 || k < numOfColsOfRow2)) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; } else { pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; @@ -1699,7 +1683,7 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, SET_DOUBLE_PTR(pData, value); break; case TSDB_DATA_TYPE_TIMESTAMP: - if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { *(TSKEY *)pData = tdGetKey(*(TKEY *)value); } else { *(TSKEY *)pData = *(TSKEY *)value; @@ -1730,8 +1714,8 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, if(forceSetNull) { while (i < numOfCols) { // the remain columns are all null data - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; } else { pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; @@ -1748,29 +1732,29 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, } } -static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, int32_t numOfCols) { - if (numOfRows == 0 || ASCENDING_TRAVERSE(pQueryHandle->order)) { +static void moveDataToFront(STsdbReadHandle* pTsdbReadHandle, int32_t numOfRows, int32_t numOfCols) { + if (numOfRows == 0 || ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { return; } // if the buffer is not full in case of descending order query, move the data in the front of the buffer - if (numOfRows < pQueryHandle->outputCapacity) { - int32_t emptySize = pQueryHandle->outputCapacity - numOfRows; + if (numOfRows < pTsdbReadHandle->outputCapacity) { + int32_t emptySize = pTsdbReadHandle->outputCapacity - numOfRows; for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); } } } -static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos, int32_t endPos, int32_t numOfExisted, +static void getQualifiedRowsPos(STsdbReadHandle* pTsdbReadHandle, int32_t startPos, int32_t endPos, int32_t numOfExisted, int32_t* start, int32_t* end) { *start = -1; - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { int32_t remain = endPos - startPos + 1; - if (remain + numOfExisted > pQueryHandle->outputCapacity) { - *end = (pQueryHandle->outputCapacity - numOfExisted) + startPos - 1; + if (remain + numOfExisted > pTsdbReadHandle->outputCapacity) { + *end = (pTsdbReadHandle->outputCapacity - numOfExisted) + startPos - 1; } else { *end = endPos; } @@ -1778,8 +1762,8 @@ static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos *start = startPos; } else { int32_t remain = (startPos - endPos) + 1; - if (remain + numOfExisted > pQueryHandle->outputCapacity) { - *end = startPos + 1 - (pQueryHandle->outputCapacity - numOfExisted); + if (remain + numOfExisted > pTsdbReadHandle->outputCapacity) { + *end = startPos + 1 - (pTsdbReadHandle->outputCapacity - numOfExisted); } else { *end = endPos; } @@ -1789,55 +1773,55 @@ static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos } } -static void updateInfoAfterMerge(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, int32_t numOfRows, int32_t endPos) { - SQueryFilePos* cur = &pQueryHandle->cur; +static void updateInfoAfterMerge(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, int32_t numOfRows, int32_t endPos) { + SQueryFilePos* cur = &pTsdbReadHandle->cur; pCheckInfo->lastKey = cur->lastKey; - pQueryHandle->realNumOfRows = numOfRows; + pTsdbReadHandle->realNumOfRows = numOfRows; cur->rows = numOfRows; cur->pos = endPos; } -static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle) { - SQueryFilePos* cur = &pQueryHandle->cur; +static void doCheckGeneratedBlockRange(STsdbReadHandle* pTsdbReadHandle) { + SQueryFilePos* cur = &pTsdbReadHandle->cur; if (cur->rows > 0) { - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - assert(cur->win.skey >= pQueryHandle->window.skey && cur->win.ekey <= pQueryHandle->window.ekey); + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { + assert(cur->win.skey >= pTsdbReadHandle->window.skey && cur->win.ekey <= pTsdbReadHandle->window.ekey); } else { - assert(cur->win.skey >= pQueryHandle->window.ekey && cur->win.ekey <= pQueryHandle->window.skey); + assert(cur->win.skey >= pTsdbReadHandle->window.ekey && cur->win.ekey <= pTsdbReadHandle->window.skey); } - SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, 0); + SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, 0); assert(cur->win.skey == ((TSKEY*)pColInfoData->pData)[0] && cur->win.ekey == ((TSKEY*)pColInfoData->pData)[cur->rows-1]); } else { - cur->win = pQueryHandle->window; + cur->win = pTsdbReadHandle->window; - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; - cur->lastKey = pQueryHandle->window.ekey + step; + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1:-1; + cur->lastKey = pTsdbReadHandle->window.ekey + step; } } -static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos) { - SQueryFilePos* cur = &pQueryHandle->cur; +static void copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos) { + SQueryFilePos* cur = &pTsdbReadHandle->cur; - SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; + SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; TSKEY* tsArray = pCols->cols[0].pData; - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; - int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1:-1; + int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pTsdbReadHandle)); int32_t pos = cur->pos; int32_t start = cur->pos; int32_t end = endPos; - if (!ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { SWAP(start, end, int32_t); } - assert(pQueryHandle->outputCapacity >= (end - start + 1)); - int32_t numOfRows = doCopyRowsFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, 0, start, end); + assert(pTsdbReadHandle->outputCapacity >= (end - start + 1)); + int32_t numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, 0, start, end); // the time window should always be ascending order: skey <= ekey cur->win = (STimeWindow) {.skey = tsArray[start], .ekey = tsArray[end]}; @@ -1846,35 +1830,34 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl cur->blockCompleted = true; // if the buffer is not full in case of descending order query, move the data in the front of the buffer - moveDataToFront(pQueryHandle, numOfRows, numOfCols); + moveDataToFront(pTsdbReadHandle, numOfRows, numOfCols); // The value of pos may be -1 or pBlockInfo->rows, and it is invalid in both cases. pos = endPos + step; - updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos); - doCheckGeneratedBlockRange(pQueryHandle); + updateInfoAfterMerge(pTsdbReadHandle, pCheckInfo, numOfRows, pos); + doCheckGeneratedBlockRange(pTsdbReadHandle); - tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, 0x%"PRIx64, - pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->mixBlock, cur->win.skey, - cur->win.ekey, cur->rows, pQueryHandle->qId); + tsdbDebug("%p uid:%" PRIu64", data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, 0x%"PRIx64, + pTsdbReadHandle, pCheckInfo->tableId, cur->mixBlock, cur->win.skey, cur->win.ekey, cur->rows, pTsdbReadHandle->qId); } -int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBlockInfo) { +int32_t getEndPosInDataBlock(STsdbReadHandle* pTsdbReadHandle, SDataBlockInfo* pBlockInfo) { // NOTE: reverse the order to find the end position in data block int32_t endPos = -1; - int32_t order = ASCENDING_TRAVERSE(pQueryHandle->order)? TSDB_ORDER_DESC : TSDB_ORDER_ASC; + int32_t order = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? TSDB_ORDER_DESC : TSDB_ORDER_ASC; - SQueryFilePos* cur = &pQueryHandle->cur; - SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; + SQueryFilePos* cur = &pTsdbReadHandle->cur; + SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; - if (ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey >= pBlockInfo->window.ekey) { + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order) && pTsdbReadHandle->window.ekey >= pBlockInfo->window.ekey) { endPos = pBlockInfo->rows - 1; cur->mixBlock = (cur->pos != 0); - } else if (!ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey <= pBlockInfo->window.skey) { + } else if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && pTsdbReadHandle->window.ekey <= pBlockInfo->window.skey) { endPos = 0; cur->mixBlock = (cur->pos != pBlockInfo->rows - 1); } else { assert(pCols->numOfRows > 0); - endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pQueryHandle->window.ekey, order); + endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pTsdbReadHandle->window.ekey, order); cur->mixBlock = true; } @@ -1883,33 +1866,33 @@ int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBl // only return the qualified data to client in terms of query time window, data rows in the same block but do not // be included in the query time window will be discarded -static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock) { - SQueryFilePos* cur = &pQueryHandle->cur; - SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); - STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; +static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock) { + SQueryFilePos* cur = &pTsdbReadHandle->cur; + SDataBlockInfo blockInfo = {0};//GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); + STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; - initTableMemIterator(pQueryHandle, pCheckInfo); + initTableMemIterator(pTsdbReadHandle, pCheckInfo); - SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; - assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == PRIMARYKEY_TIMESTAMP_COL_INDEX && + SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; + assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == PRIMARYKEY_TIMESTAMP_COL_ID && cur->pos >= 0 && cur->pos < pBlock->numOfRows); TSKEY* tsArray = pCols->cols[0].pData; assert(pCols->numOfRows == pBlock->numOfRows && tsArray[0] == pBlock->keyFirst && tsArray[pBlock->numOfRows-1] == pBlock->keyLast); // for search the endPos, so the order needs to reverse - int32_t order = (pQueryHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; + int32_t order = (pTsdbReadHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; - int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1:-1; + int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pTsdbReadHandle)); STable* pTable = pCheckInfo->pTableObj; - int32_t endPos = getEndPosInDataBlock(pQueryHandle, &blockInfo); + int32_t endPos = getEndPosInDataBlock(pTsdbReadHandle, &blockInfo); - tsdbDebug("%p uid:%" PRIu64",tid:%d start merge data block, file block range:%"PRIu64"-%"PRIu64" rows:%d, start:%d," + tsdbDebug("%p uid:%" PRIu64" start merge data block, file block range:%"PRIu64"-%"PRIu64" rows:%d, start:%d," "end:%d, 0x%"PRIx64, - pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, blockInfo.window.skey, blockInfo.window.ekey, - blockInfo.rows, cur->pos, endPos, pQueryHandle->qId); + pTsdbReadHandle, pCheckInfo->tableId, blockInfo.window.skey, blockInfo.window.ekey, + blockInfo.rows, cur->pos, endPos, pTsdbReadHandle->qId); // compared with the data from in-memory buffer, to generate the correct timestamp array list int32_t numOfRows = 0; @@ -1924,40 +1907,40 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* // no data in buffer, load data from file directly if (pCheckInfo->iiter == NULL && pCheckInfo->iter == NULL) { - copyAllRemainRowsFromFileBlock(pQueryHandle, pCheckInfo, &blockInfo, endPos); + copyAllRemainRowsFromFileBlock(pTsdbReadHandle, pCheckInfo, &blockInfo, endPos); return; } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) { SSkipListNode* node = NULL; do { SMemRow row2 = NULL; - SMemRow row1 = getSMemRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update, &row2); + SMemRow row1 = getSMemRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, &row2); if (row1 == NULL) { break; } TSKEY key = memRowKey(row1); - if ((key > pQueryHandle->window.ekey && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (key < pQueryHandle->window.ekey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + if ((key > pTsdbReadHandle->window.ekey && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + (key < pTsdbReadHandle->window.ekey && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { break; } - if (((pos > endPos || tsArray[pos] > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || - ((pos < endPos || tsArray[pos] < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + if (((pos > endPos || tsArray[pos] > pTsdbReadHandle->window.ekey) && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + ((pos < endPos || tsArray[pos] < pTsdbReadHandle->window.ekey) && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { break; } - if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (key > tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + (key > tsArray[pos] && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { if (rv1 != memRowVersion(row1)) { - pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); +// pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); rv1 = memRowVersion(row1); } if(row2 && rv2 != memRowVersion(row2)) { - pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); +// pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); rv2 = memRowVersion(row2); } - mergeTwoRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row1, row2, numOfCols, pTable, pSchema1, pSchema2, true); + mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, row1, row2, numOfCols, pTable, pSchema1, pSchema2, true); numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -1971,19 +1954,19 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it if (pCfg->update) { if(pCfg->update == TD_ROW_PARTIAL_UPDATE) { - doCopyRowsFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, pos, pos); + doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, pos, pos); } if (rv1 != memRowVersion(row1)) { - pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); +// pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); rv1 = memRowVersion(row1); } if(row2 && rv2 != memRowVersion(row2)) { - pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); +// pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); rv2 = memRowVersion(row2); } bool forceSetNull = pCfg->update != TD_ROW_PARTIAL_UPDATE; - mergeTwoRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row1, row2, numOfCols, pTable, pSchema1, pSchema2, forceSetNull); + mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, row1, row2, numOfCols, pTable, pSchema1, pSchema2, forceSetNull); numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -1998,8 +1981,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } else { moveToNextRowInMem(pCheckInfo); } - } else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (key < tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + } else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + (key < tsArray[pos] && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = tsArray[pos]; } @@ -2016,38 +1999,38 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } int32_t qstart = 0, qend = 0; - getQualifiedRowsPos(pQueryHandle, pos, end, numOfRows, &qstart, &qend); + getQualifiedRowsPos(pTsdbReadHandle, pos, end, numOfRows, &qstart, &qend); - numOfRows = doCopyRowsFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, qstart, qend); + numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, qstart, qend); pos += (qend - qstart + 1) * step; - cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? tsArray[qend]:tsArray[qstart]; + cur->win.ekey = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? tsArray[qend]:tsArray[qstart]; cur->lastKey = cur->win.ekey + step; } - } while (numOfRows < pQueryHandle->outputCapacity); + } while (numOfRows < pTsdbReadHandle->outputCapacity); - if (numOfRows < pQueryHandle->outputCapacity) { + if (numOfRows < pTsdbReadHandle->outputCapacity) { /** * if cache is empty, load remain file block data. In contrast, if there are remain data in cache, do NOT * copy them all to result buffer, since it may be overlapped with file data block. */ if (node == NULL || - ((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && - ASCENDING_TRAVERSE(pQueryHandle->order)) || - ((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && - !ASCENDING_TRAVERSE(pQueryHandle->order))) { + ((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) > pTsdbReadHandle->window.ekey) && + ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + ((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) < pTsdbReadHandle->window.ekey) && + !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { // no data in cache or data in cache is greater than the ekey of time window, load data from file block if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = tsArray[pos]; } int32_t start = -1, end = -1; - getQualifiedRowsPos(pQueryHandle, pos, endPos, numOfRows, &start, &end); + getQualifiedRowsPos(pTsdbReadHandle, pos, endPos, numOfRows, &start, &end); - numOfRows = doCopyRowsFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end); + numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, start, end); pos += (end - start + 1) * step; - cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? tsArray[end]:tsArray[start]; + cur->win.ekey = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? tsArray[end]:tsArray[start]; cur->lastKey = cur->win.ekey + step; cur->mixBlock = true; } @@ -2055,20 +2038,19 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } cur->blockCompleted = - (((pos > endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || - ((pos < endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))); + (((pos > endPos || cur->lastKey > pTsdbReadHandle->window.ekey) && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + ((pos < endPos || cur->lastKey < pTsdbReadHandle->window.ekey) && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))); - if (!ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { SWAP(cur->win.skey, cur->win.ekey, TSKEY); } - moveDataToFront(pQueryHandle, numOfRows, numOfCols); - updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos); - doCheckGeneratedBlockRange(pQueryHandle); + moveDataToFront(pTsdbReadHandle, numOfRows, numOfCols); + updateInfoAfterMerge(pTsdbReadHandle, pCheckInfo, numOfRows, pos); + doCheckGeneratedBlockRange(pTsdbReadHandle); - tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, 0x%"PRIx64, - pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->mixBlock, cur->win.skey, - cur->win.ekey, cur->rows, pQueryHandle->qId); + tsdbDebug("%p uid:%" PRIu64", data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, 0x%"PRIx64, + pTsdbReadHandle, pCheckInfo->tableId, cur->mixBlock, cur->win.skey, cur->win.ekey, cur->rows, pTsdbReadHandle->qId); } int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { @@ -2174,24 +2156,24 @@ static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void* return pLeftBlockInfoEx->compBlock->offset > pRightBlockInfoEx->compBlock->offset ? 1 : -1; } -static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numOfBlocks, int32_t* numOfAllocBlocks) { +static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t numOfBlocks, int32_t* numOfAllocBlocks) { size_t size = sizeof(STableBlockInfo) * numOfBlocks; - if (pQueryHandle->allocSize < size) { - pQueryHandle->allocSize = (int32_t)size; - char* tmp = realloc(pQueryHandle->pDataBlockInfo, pQueryHandle->allocSize); + if (pTsdbReadHandle->allocSize < size) { + pTsdbReadHandle->allocSize = (int32_t)size; + char* tmp = realloc(pTsdbReadHandle->pDataBlockInfo, pTsdbReadHandle->allocSize); if (tmp == NULL) { return TSDB_CODE_TDB_OUT_OF_MEMORY; } - pQueryHandle->pDataBlockInfo = (STableBlockInfo*) tmp; + pTsdbReadHandle->pDataBlockInfo = (STableBlockInfo*) tmp; } - memset(pQueryHandle->pDataBlockInfo, 0, size); + memset(pTsdbReadHandle->pDataBlockInfo, 0, size); *numOfAllocBlocks = numOfBlocks; // access data blocks according to the offset of each block in asc/desc order. - int32_t numOfTables = (int32_t)taosArrayGetSize(pQueryHandle->pTableCheckInfo); + int32_t numOfTables = (int32_t)taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); SBlockOrderSupporter sup = {0}; sup.numOfTables = numOfTables; @@ -2208,7 +2190,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO int32_t numOfQualTables = 0; for (int32_t j = 0; j < numOfTables; ++j) { - STableCheckInfo* pTableCheck = (STableCheckInfo*)taosArrayGet(pQueryHandle->pTableCheckInfo, j); + STableCheckInfo* pTableCheck = (STableCheckInfo*)taosArrayGet(pTsdbReadHandle->pTableCheckInfo, j); if (pTableCheck->numOfBlocks <= 0) { continue; } @@ -2239,16 +2221,16 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO // since there is only one table qualified, blocks are not sorted if (numOfQualTables == 1) { - memcpy(pQueryHandle->pDataBlockInfo, sup.pDataBlockInfo[0], sizeof(STableBlockInfo) * numOfBlocks); + memcpy(pTsdbReadHandle->pDataBlockInfo, sup.pDataBlockInfo[0], sizeof(STableBlockInfo) * numOfBlocks); cleanBlockOrderSupporter(&sup, numOfQualTables); - tsdbDebug("%p create data blocks info struct completed for 1 table, %d blocks not sorted 0x%"PRIx64, pQueryHandle, cnt, - pQueryHandle->qId); + tsdbDebug("%p create data blocks info struct completed for 1 table, %d blocks not sorted 0x%"PRIx64, pTsdbReadHandle, cnt, + pTsdbReadHandle->qId); return TSDB_CODE_SUCCESS; } - tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables 0x%"PRIx64, pQueryHandle, cnt, - numOfQualTables, pQueryHandle->qId); + tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables 0x%"PRIx64, pTsdbReadHandle, cnt, + numOfQualTables, pTsdbReadHandle->qId); assert(cnt <= numOfBlocks && numOfQualTables <= numOfTables); // the pTableQueryInfo[j]->numOfBlocks may be 0 sup.numOfTables = numOfQualTables; @@ -2267,7 +2249,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO int32_t index = sup.blockIndexArray[pos]++; STableBlockInfo* pBlocksInfo = sup.pDataBlockInfo[pos]; - pQueryHandle->pDataBlockInfo[numOfTotal++] = pBlocksInfo[index]; + pTsdbReadHandle->pDataBlockInfo[numOfTotal++] = pBlocksInfo[index]; // set data block index overflow, in order to disable the offset comparator if (sup.blockIndexArray[pos] >= sup.numOfBlocksPerTable[pos]) { @@ -2284,90 +2266,90 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO * } */ - tsdbDebug("%p %d data blocks sort completed, 0x%"PRIx64, pQueryHandle, cnt, pQueryHandle->qId); + tsdbDebug("%p %d data blocks sort completed, 0x%"PRIx64, pTsdbReadHandle, cnt, pTsdbReadHandle->qId); cleanBlockOrderSupporter(&sup, numOfTables); free(pTree); return TSDB_CODE_SUCCESS; } -static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exists); +static int32_t getFirstFileDataBlock(STsdbReadHandle* pTsdbReadHandle, bool* exists); -static int32_t getDataBlockRv(STsdbQueryHandle* pQueryHandle, STableBlockInfo* pNext, bool *exists) { - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; - SQueryFilePos* cur = &pQueryHandle->cur; +static int32_t getDataBlockRv(STsdbReadHandle* pTsdbReadHandle, STableBlockInfo* pNext, bool *exists) { + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1 : -1; + SQueryFilePos* cur = &pTsdbReadHandle->cur; while(1) { - int32_t code = loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo, exists); + int32_t code = loadFileDataBlock(pTsdbReadHandle, pNext->compBlock, pNext->pTableCheckInfo, exists); if (code != TSDB_CODE_SUCCESS || *exists) { return code; } - if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (cur->slot == 0 && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + if ((cur->slot == pTsdbReadHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + (cur->slot == 0 && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { // all data blocks in current file has been checked already, try next file if exists - return getFirstFileDataBlock(pQueryHandle, exists); + return getFirstFileDataBlock(pTsdbReadHandle, exists); } else { // next block of the same file cur->slot += step; cur->mixBlock = false; cur->blockCompleted = false; - pNext = &pQueryHandle->pDataBlockInfo[cur->slot]; + pNext = &pTsdbReadHandle->pDataBlockInfo[cur->slot]; } } } -static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exists) { - pQueryHandle->numOfBlocks = 0; - SQueryFilePos* cur = &pQueryHandle->cur; +static int32_t getFirstFileDataBlock(STsdbReadHandle* pTsdbReadHandle, bool* exists) { + pTsdbReadHandle->numOfBlocks = 0; + SQueryFilePos* cur = &pTsdbReadHandle->cur; int32_t code = TSDB_CODE_SUCCESS; int32_t numOfBlocks = 0; - int32_t numOfTables = (int32_t)taosArrayGetSize(pQueryHandle->pTableCheckInfo); + int32_t numOfTables = (int32_t)taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); - STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; + STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; STimeWindow win = TSWINDOW_INITIALIZER; while (true) { - tsdbRLockFS(REPO_FS(pQueryHandle->pTsdb)); + tsdbRLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); - if ((pQueryHandle->pFileGroup = tsdbFSIterNext(&pQueryHandle->fileIter)) == NULL) { - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + if ((pTsdbReadHandle->pFileGroup = tsdbFSIterNext(&pTsdbReadHandle->fileIter)) == NULL) { + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); break; } - tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pQueryHandle->pFileGroup->fid, &win.skey, &win.ekey); + tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pTsdbReadHandle->pFileGroup->fid, &win.skey, &win.ekey); // current file are not overlapped with query time window, ignore remain files - if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) || - (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) { - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); - tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pQueryHandle, - pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qId); - pQueryHandle->pFileGroup = NULL; - assert(pQueryHandle->numOfBlocks == 0); + if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.skey > pTsdbReadHandle->window.ekey) || + (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.ekey < pTsdbReadHandle->window.ekey)) { + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); + tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pTsdbReadHandle, + pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, pTsdbReadHandle->qId); + pTsdbReadHandle->pFileGroup = NULL; + assert(pTsdbReadHandle->numOfBlocks == 0); break; } - if (tsdbSetAndOpenReadFSet(&pQueryHandle->rhelper, pQueryHandle->pFileGroup) < 0) { - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + if (tsdbSetAndOpenReadFSet(&pTsdbReadHandle->rhelper, pTsdbReadHandle->pFileGroup) < 0) { + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); code = terrno; break; } - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); - if (tsdbLoadBlockIdx(&pQueryHandle->rhelper) < 0) { + if (tsdbLoadBlockIdx(&pTsdbReadHandle->rhelper) < 0) { code = terrno; break; } - if ((code = getFileCompInfo(pQueryHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) { + if ((code = getFileCompInfo(pTsdbReadHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) { break; } - tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, 0x%"PRIx64, pQueryHandle, numOfBlocks, numOfTables, - pQueryHandle->pFileGroup->fid, pQueryHandle->qId); + tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, 0x%"PRIx64, pTsdbReadHandle, numOfBlocks, numOfTables, + pTsdbReadHandle->pFileGroup->fid, pTsdbReadHandle->qId); assert(numOfBlocks >= 0); if (numOfBlocks == 0) { @@ -2375,20 +2357,20 @@ static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exist } // todo return error code to query engine - if ((code = createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks)) != TSDB_CODE_SUCCESS) { + if ((code = createDataBlocksInfo(pTsdbReadHandle, numOfBlocks, &pTsdbReadHandle->numOfBlocks)) != TSDB_CODE_SUCCESS) { break; } - assert(numOfBlocks >= pQueryHandle->numOfBlocks); - if (pQueryHandle->numOfBlocks > 0) { + assert(numOfBlocks >= pTsdbReadHandle->numOfBlocks); + if (pTsdbReadHandle->numOfBlocks > 0) { break; } } // no data in file anymore - if (pQueryHandle->numOfBlocks <= 0 || code != TSDB_CODE_SUCCESS) { + if (pTsdbReadHandle->numOfBlocks <= 0 || code != TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) { - assert(pQueryHandle->pFileGroup == NULL); + assert(pTsdbReadHandle->pFileGroup == NULL); } cur->fid = INT32_MIN; // denote that there are no data in file anymore @@ -2396,12 +2378,12 @@ static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exist return code; } - assert(pQueryHandle->pFileGroup != NULL && pQueryHandle->numOfBlocks > 0); - cur->slot = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1; - cur->fid = pQueryHandle->pFileGroup->fid; + assert(pTsdbReadHandle->pFileGroup != NULL && pTsdbReadHandle->numOfBlocks > 0); + cur->slot = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 0:pTsdbReadHandle->numOfBlocks-1; + cur->fid = pTsdbReadHandle->pFileGroup->fid; - STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; - return getDataBlockRv(pQueryHandle, pBlockInfo, exists); + STableBlockInfo* pBlockInfo = &pTsdbReadHandle->pDataBlockInfo[cur->slot]; + return getDataBlockRv(pTsdbReadHandle, pBlockInfo, exists); } static bool isEndFileDataBlock(SQueryFilePos* cur, int32_t numOfBlocks, bool ascTrav) { @@ -2409,90 +2391,90 @@ static bool isEndFileDataBlock(SQueryFilePos* cur, int32_t numOfBlocks, bool asc return (cur->slot == numOfBlocks - 1 && ascTrav) || (cur->slot == 0 && !ascTrav); } -static void moveToNextDataBlockInCurrentFile(STsdbQueryHandle* pQueryHandle) { - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; +static void moveToNextDataBlockInCurrentFile(STsdbReadHandle* pTsdbReadHandle) { + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1 : -1; - SQueryFilePos* cur = &pQueryHandle->cur; - assert(cur->slot < pQueryHandle->numOfBlocks && cur->slot >= 0); + SQueryFilePos* cur = &pTsdbReadHandle->cur; + assert(cur->slot < pTsdbReadHandle->numOfBlocks && cur->slot >= 0); cur->slot += step; cur->mixBlock = false; cur->blockCompleted = false; } - -int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist* pTableBlockInfo) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) queryHandle; +#if 0 +int32_t tsdbGetFileBlocksDistInfo(tsdbReadHandleT* queryHandle, STableBlockDist* pTableBlockInfo) { + STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) queryHandle; pTableBlockInfo->totalSize = 0; pTableBlockInfo->totalRows = 0; - STsdbFS* pFileHandle = REPO_FS(pQueryHandle->pTsdb); + STsdbFS* pFileHandle = REPO_FS(pTsdbReadHandle->pTsdb); // find the start data block in file - pQueryHandle->locateStart = true; - STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; - int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision); + pTsdbReadHandle->locateStart = true; + STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; + int32_t fid = getFileIdFromKey(pTsdbReadHandle->window.skey, pCfg->daysPerFile, pCfg->precision); tsdbRLockFS(pFileHandle); - tsdbFSIterInit(&pQueryHandle->fileIter, pFileHandle, pQueryHandle->order); - tsdbFSIterSeek(&pQueryHandle->fileIter, fid); + tsdbFSIterInit(&pTsdbReadHandle->fileIter, pFileHandle, pTsdbReadHandle->order); + tsdbFSIterSeek(&pTsdbReadHandle->fileIter, fid); tsdbUnLockFS(pFileHandle); pTableBlockInfo->numOfFiles += 1; int32_t code = TSDB_CODE_SUCCESS; int32_t numOfBlocks = 0; - int32_t numOfTables = (int32_t)taosArrayGetSize(pQueryHandle->pTableCheckInfo); + int32_t numOfTables = (int32_t)taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); int defaultRows = TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock); STimeWindow win = TSWINDOW_INITIALIZER; while (true) { numOfBlocks = 0; - tsdbRLockFS(REPO_FS(pQueryHandle->pTsdb)); + tsdbRLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); - if ((pQueryHandle->pFileGroup = tsdbFSIterNext(&pQueryHandle->fileIter)) == NULL) { - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + if ((pTsdbReadHandle->pFileGroup = tsdbFSIterNext(&pTsdbReadHandle->fileIter)) == NULL) { + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); break; } - tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pQueryHandle->pFileGroup->fid, &win.skey, &win.ekey); + tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pTsdbReadHandle->pFileGroup->fid, &win.skey, &win.ekey); // current file are not overlapped with query time window, ignore remain files - if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) || - (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) { - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); - tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pQueryHandle, - pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qId); - pQueryHandle->pFileGroup = NULL; + if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.skey > pTsdbReadHandle->window.ekey) || + (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.ekey < pTsdbReadHandle->window.ekey)) { + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); + tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pTsdbReadHandle, + pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, pTsdbReadHandle->qId); + pTsdbReadHandle->pFileGroup = NULL; break; } pTableBlockInfo->numOfFiles += 1; - if (tsdbSetAndOpenReadFSet(&pQueryHandle->rhelper, pQueryHandle->pFileGroup) < 0) { - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + if (tsdbSetAndOpenReadFSet(&pTsdbReadHandle->rhelper, pTsdbReadHandle->pFileGroup) < 0) { + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); code = terrno; break; } - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); - if (tsdbLoadBlockIdx(&pQueryHandle->rhelper) < 0) { + if (tsdbLoadBlockIdx(&pTsdbReadHandle->rhelper) < 0) { code = terrno; break; } - if ((code = getFileCompInfo(pQueryHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) { + if ((code = getFileCompInfo(pTsdbReadHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) { break; } - tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, 0x%"PRIx64, pQueryHandle, numOfBlocks, numOfTables, - pQueryHandle->pFileGroup->fid, pQueryHandle->qId); + tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, 0x%"PRIx64, pTsdbReadHandle, numOfBlocks, numOfTables, + pTsdbReadHandle->pFileGroup->fid, pTsdbReadHandle->qId); if (numOfBlocks == 0) { continue; } for (int32_t i = 0; i < numOfTables; ++i) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); SBlock* pBlock = pCheckInfo->pCompInfo->blocks; for (int32_t j = 0; j < pCheckInfo->numOfBlocks; ++j) { @@ -2512,36 +2494,37 @@ int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist return code; } +#endif -static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists) { - STsdbFS* pFileHandle = REPO_FS(pQueryHandle->pTsdb); - SQueryFilePos* cur = &pQueryHandle->cur; +static int32_t getDataBlocksInFiles(STsdbReadHandle* pTsdbReadHandle, bool* exists) { + STsdbFS* pFileHandle = REPO_FS(pTsdbReadHandle->pTsdb); + SQueryFilePos* cur = &pTsdbReadHandle->cur; // find the start data block in file - if (!pQueryHandle->locateStart) { - pQueryHandle->locateStart = true; - STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; - int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision); + if (!pTsdbReadHandle->locateStart) { + pTsdbReadHandle->locateStart = true; + STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; + int32_t fid = getFileIdFromKey(pTsdbReadHandle->window.skey, pCfg->daysPerFile, pCfg->precision); tsdbRLockFS(pFileHandle); - tsdbFSIterInit(&pQueryHandle->fileIter, pFileHandle, pQueryHandle->order); - tsdbFSIterSeek(&pQueryHandle->fileIter, fid); + tsdbFSIterInit(&pTsdbReadHandle->fileIter, pFileHandle, pTsdbReadHandle->order); + tsdbFSIterSeek(&pTsdbReadHandle->fileIter, fid); tsdbUnLockFS(pFileHandle); - return getFirstFileDataBlock(pQueryHandle, exists); + return getFirstFileDataBlock(pTsdbReadHandle, exists); } else { // check if current file block is all consumed - STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; + STableBlockInfo* pBlockInfo = &pTsdbReadHandle->pDataBlockInfo[cur->slot]; STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo; // current block is done, try next if ((!cur->mixBlock) || cur->blockCompleted) { // all data blocks in current file has been checked already, try next file if exists } else { - tsdbDebug("%p continue in current data block, index:%d, pos:%d, 0x%"PRIx64, pQueryHandle, cur->slot, cur->pos, - pQueryHandle->qId); - int32_t code = handleDataMergeIfNeeded(pQueryHandle, pBlockInfo->compBlock, pCheckInfo); - *exists = (pQueryHandle->realNumOfRows > 0); + tsdbDebug("%p continue in current data block, index:%d, pos:%d, 0x%"PRIx64, pTsdbReadHandle, cur->slot, cur->pos, + pTsdbReadHandle->qId); + int32_t code = handleDataMergeIfNeeded(pTsdbReadHandle, pBlockInfo->compBlock, pCheckInfo); + *exists = (pTsdbReadHandle->realNumOfRows > 0); if (code != TSDB_CODE_SUCCESS || *exists) { return code; @@ -2550,50 +2533,50 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists // current block is empty, try next block in file // all data blocks in current file has been checked already, try next file if exists - if (isEndFileDataBlock(cur, pQueryHandle->numOfBlocks, ASCENDING_TRAVERSE(pQueryHandle->order))) { - return getFirstFileDataBlock(pQueryHandle, exists); + if (isEndFileDataBlock(cur, pTsdbReadHandle->numOfBlocks, ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { + return getFirstFileDataBlock(pTsdbReadHandle, exists); } else { - moveToNextDataBlockInCurrentFile(pQueryHandle); - STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot]; - return getDataBlockRv(pQueryHandle, pNext, exists); + moveToNextDataBlockInCurrentFile(pTsdbReadHandle); + STableBlockInfo* pNext = &pTsdbReadHandle->pDataBlockInfo[cur->slot]; + return getDataBlockRv(pTsdbReadHandle, pNext, exists); } } } -static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) { - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); +static bool doHasDataInBuffer(STsdbReadHandle* pTsdbReadHandle) { + size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); - while (pQueryHandle->activeIndex < numOfTables) { - if (hasMoreDataInCache(pQueryHandle)) { + while (pTsdbReadHandle->activeIndex < numOfTables) { + if (hasMoreDataInCache(pTsdbReadHandle)) { return true; } - pQueryHandle->activeIndex += 1; + pTsdbReadHandle->activeIndex += 1; } // no data in memtable or imemtable, decrease the memory reference. // TODO !! -// tsdbMayUnTakeMemSnapshot(pQueryHandle); +// tsdbMayUnTakeMemSnapshot(pTsdbReadHandle); return false; } //todo not unref yet, since it is not support multi-group interpolation query -static UNUSED_FUNC void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { +static UNUSED_FUNC void changeQueryHandleForInterpQuery(tsdbReadHandleT pHandle) { // filter the queried time stamp in the first place - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; + STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; // starts from the buffer in case of descending timestamp order check data blocks - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); int32_t i = 0; while(i < numOfTables) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); // the first qualified table for interpolation query - if ((pQueryHandle->window.skey <= pCheckInfo->pTableObj->lastKey) && - (pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL)) { - break; - } +// if ((pTsdbReadHandle->window.skey <= pCheckInfo->pTableObj->lastKey) && +// (pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL)) { +// break; +// } i++; } @@ -2603,35 +2586,34 @@ static UNUSED_FUNC void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle return; } - STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i); - taosArrayClear(pQueryHandle->pTableCheckInfo); + STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); + taosArrayClear(pTsdbReadHandle->pTableCheckInfo); - info.lastKey = pQueryHandle->window.skey; - taosArrayPush(pQueryHandle->pTableCheckInfo, &info); + info.lastKey = pTsdbReadHandle->window.skey; + taosArrayPush(pTsdbReadHandle->pTableCheckInfo, &info); } static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, - STsdbQueryHandle* pQueryHandle) { + STsdbReadHandle* pTsdbReadHandle) { int numOfRows = 0; - int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns); - STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; + int32_t numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns); + STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; win->skey = TSKEY_INITIAL_VAL; int64_t st = taosGetTimestampUs(); - STable* pTable = pCheckInfo->pTableObj; int16_t rv = -1; STSchema* pSchema = NULL; do { - SMemRow row = getSMemRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update, NULL); + SMemRow row = getSMemRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, NULL); if (row == NULL) { break; } TSKEY key = memRowKey(row); - if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { - tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey, - pQueryHandle->window.ekey); + if ((key > maxKey && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { + tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pTsdbReadHandle, key, pTsdbReadHandle->window.skey, + pTsdbReadHandle->window.ekey); break; } @@ -2642,10 +2624,10 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int win->ekey = key; if (rv != memRowVersion(row)) { - pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row)); + pSchema = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, pCheckInfo->tableId, 0); rv = memRowVersion(row); } - mergeTwoRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, NULL, numOfCols, pTable, pSchema, NULL, true); + mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, numOfRows, row, NULL, numOfCols, pCheckInfo->tableId, pSchema, NULL, true); if (++numOfRows >= maxRowsToRead) { moveToNextRowInMem(pCheckInfo); @@ -2657,24 +2639,24 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int assert(numOfRows <= maxRowsToRead); // if the buffer is not full in case of descending order query, move the data in the front of the buffer - if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < maxRowsToRead) { + if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && numOfRows < maxRowsToRead) { int32_t emptySize = maxRowsToRead - numOfRows; for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); } } int64_t elapsedTime = taosGetTimestampUs() - st; - tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, 0x%"PRIx64, pQueryHandle, - elapsedTime, numOfRows, numOfCols, pQueryHandle->qId); + tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, 0x%"PRIx64, pTsdbReadHandle, + elapsedTime, numOfRows, numOfCols, pTsdbReadHandle->qId); return numOfRows; } static int32_t getAllTableList(STable* pSuperTable, SArray* list) { - SSkipListIterator* iter = tSkipListCreateIter(pSuperTable->pIndex); + SSkipListIterator* iter = NULL;//tSkipListCreateIter(pSuperTable->pIndex); while (tSkipListIterNext(iter)) { SSkipListNode* pNode = tSkipListIterGet(iter); @@ -2693,58 +2675,61 @@ static void destroyHelper(void* param) { return; } - tQueryInfo* pInfo = (tQueryInfo*)param; - if (pInfo->optr != TSDB_RELATION_IN) { - tfree(pInfo->q); - } else { - taosHashCleanup((SHashObj *)(pInfo->q)); - } +// tQueryInfo* pInfo = (tQueryInfo*)param; +// if (pInfo->optr != TSDB_RELATION_IN) { +// tfree(pInfo->q); +// } else { +// taosHashCleanup((SHashObj *)(pInfo->q)); +// } free(param); } -static bool loadBlockOfActiveTable(STsdbQueryHandle* pQueryHandle) { - if (pQueryHandle->checkFiles) { +#define TSDB_PREV_ROW 0x1 +#define TSDB_NEXT_ROW 0x2 + +static bool loadBlockOfActiveTable(STsdbReadHandle* pTsdbReadHandle) { + if (pTsdbReadHandle->checkFiles) { // check if the query range overlaps with the file data block bool exists = true; - int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); + int32_t code = getDataBlocksInFiles(pTsdbReadHandle, &exists); if (code != TSDB_CODE_SUCCESS) { - pQueryHandle->checkFiles = false; + pTsdbReadHandle->checkFiles = false; return false; } if (exists) { - tsdbRetrieveDataBlock((TsdbQueryHandleT*) pQueryHandle, NULL); - if (pQueryHandle->currentLoadExternalRows && pQueryHandle->window.skey == pQueryHandle->window.ekey) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, 0); - assert(*(int64_t*)pColInfo->pData == pQueryHandle->window.skey); + tsdbRetrieveDataBlock((tsdbReadHandleT*) pTsdbReadHandle, NULL); + if (pTsdbReadHandle->currentLoadExternalRows && pTsdbReadHandle->window.skey == pTsdbReadHandle->window.ekey) { + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, 0); + assert(*(int64_t*)pColInfo->pData == pTsdbReadHandle->window.skey); } - pQueryHandle->currentLoadExternalRows = false; // clear the flag, since the exact matched row is found. + pTsdbReadHandle->currentLoadExternalRows = false; // clear the flag, since the exact matched row is found. return exists; } - pQueryHandle->checkFiles = false; + pTsdbReadHandle->checkFiles = false; } - if (hasMoreDataInCache(pQueryHandle)) { - pQueryHandle->currentLoadExternalRows = false; + if (hasMoreDataInCache(pTsdbReadHandle)) { + pTsdbReadHandle->currentLoadExternalRows = false; return true; } // current result is empty - if (pQueryHandle->currentLoadExternalRows && pQueryHandle->window.skey == pQueryHandle->window.ekey && pQueryHandle->cur.rows == 0) { - SMemRef* pMemRef = pQueryHandle->pMemRef; + if (pTsdbReadHandle->currentLoadExternalRows && pTsdbReadHandle->window.skey == pTsdbReadHandle->window.ekey && pTsdbReadHandle->cur.rows == 0) { +// STsdbMemTable* pMemRef = pTsdbReadHandle->pMemTable; - doGetExternalRow(pQueryHandle, TSDB_PREV_ROW, pMemRef); - doGetExternalRow(pQueryHandle, TSDB_NEXT_ROW, pMemRef); +// doGetExternalRow(pTsdbReadHandle, TSDB_PREV_ROW, pMemRef); +// doGetExternalRow(pTsdbReadHandle, TSDB_NEXT_ROW, pMemRef); - bool result = tsdbGetExternalRow(pQueryHandle); + bool result = tsdbGetExternalRow(pTsdbReadHandle); - pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); - pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); - pQueryHandle->currentLoadExternalRows = false; +// pTsdbReadHandle->prev = doFreeColumnInfoData(pTsdbReadHandle->prev); +// pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next); + pTsdbReadHandle->currentLoadExternalRows = false; return result; } @@ -2752,26 +2737,26 @@ static bool loadBlockOfActiveTable(STsdbQueryHandle* pQueryHandle) { return false; } -static bool loadCachedLastRow(STsdbQueryHandle* pQueryHandle) { +static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { // the last row is cached in buffer, return it directly. - // here note that the pQueryHandle->window must be the TS_INITIALIZER - int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + // here note that the pTsdbReadHandle->window must be the TS_INITIALIZER + int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pTsdbReadHandle)); + size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); assert(numOfTables > 0 && numOfCols > 0); - SQueryFilePos* cur = &pQueryHandle->cur; + SQueryFilePos* cur = &pTsdbReadHandle->cur; SMemRow pRow = NULL; TSKEY key = TSKEY_INITIAL_VAL; - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; - - if (++pQueryHandle->activeIndex < numOfTables) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); - int32_t ret = tsdbGetCachedLastRow(pCheckInfo->pTableObj, &pRow, &key); - if (ret != TSDB_CODE_SUCCESS) { - return false; - } - mergeTwoRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, 0, pRow, NULL, numOfCols, pCheckInfo->pTableObj, NULL, NULL, true); + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1:-1; + + if (++pTsdbReadHandle->activeIndex < numOfTables) { + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex); +// int32_t ret = tsdbGetCachedLastRow(pCheckInfo->pTableObj, &pRow, &key); +// if (ret != TSDB_CODE_SUCCESS) { +// return false; +// } + mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, 0, pRow, NULL, numOfCols, pCheckInfo->pTableObj, NULL, NULL, true); tfree(pRow); // update the last key value @@ -2791,192 +2776,191 @@ static bool loadCachedLastRow(STsdbQueryHandle* pQueryHandle) { -static bool loadCachedLast(STsdbQueryHandle* pQueryHandle) { - // the last row is cached in buffer, return it directly. - // here note that the pQueryHandle->window must be the TS_INITIALIZER - int32_t tgNumOfCols = (int32_t)QH_GET_NUM_OF_COLS(pQueryHandle); - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - int32_t numOfRows = 0; - assert(numOfTables > 0 && tgNumOfCols > 0); - SQueryFilePos* cur = &pQueryHandle->cur; - TSKEY priKey = TSKEY_INITIAL_VAL; - int32_t priIdx = -1; - SColumnInfoData* pColInfo = NULL; - - while (++pQueryHandle->activeIndex < numOfTables) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); - STable* pTable = pCheckInfo->pTableObj; - char* pData = NULL; - - int32_t numOfCols = pTable->maxColNum; - - if (pTable->lastCols == NULL || pTable->maxColNum <= 0) { - tsdbWarn("no last cached for table %s, uid:%" PRIu64 ",tid:%d", pTable->name->data, pTable->tableId.uid, pTable->tableId.tid); - continue; - } - - int32_t i = 0, j = 0; - while(i < tgNumOfCols && j < numOfCols) { - pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - if (pTable->lastCols[j].colId < pColInfo->info.colId) { - j++; - continue; - } else if (pTable->lastCols[j].colId > pColInfo->info.colId) { - i++; - continue; - } - - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; - - if (pTable->lastCols[j].bytes > 0) { - void* value = pTable->lastCols[j].pData; - switch (pColInfo->info.type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - memcpy(pData, value, varDataTLen(value)); - break; - case TSDB_DATA_TYPE_NULL: - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - *(uint8_t *)pData = *(uint8_t *)value; - break; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - *(uint16_t *)pData = *(uint16_t *)value; - break; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - *(uint32_t *)pData = *(uint32_t *)value; - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - *(uint64_t *)pData = *(uint64_t *)value; - break; - case TSDB_DATA_TYPE_FLOAT: - SET_FLOAT_PTR(pData, value); - break; - case TSDB_DATA_TYPE_DOUBLE: - SET_DOUBLE_PTR(pData, value); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - priKey = tdGetKey(*(TKEY *)value); - priIdx = i; - - i++; - j++; - continue; - } else { - *(TSKEY *)pData = *(TSKEY *)value; - } - break; - default: - memcpy(pData, value, pColInfo->info.bytes); - } - - for (int32_t n = 0; n < tgNumOfCols; ++n) { - if (n == i) { - continue; - } - - pColInfo = taosArrayGet(pQueryHandle->pColumns, n); - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; - - if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - *(TSKEY *)pData = pTable->lastCols[j].ts; - continue; - } - - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(pData, pColInfo->info.type); - } else { - setNull(pData, pColInfo->info.type, pColInfo->info.bytes); - } - } - - numOfRows++; - assert(numOfRows < pQueryHandle->outputCapacity); - } - - i++; - j++; - } - - // leave the real ts column as the last row, because last function only (not stable) use the last row as res - if (priKey != TSKEY_INITIAL_VAL) { - pColInfo = taosArrayGet(pQueryHandle->pColumns, priIdx); - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; - - *(TSKEY *)pData = priKey; - - for (int32_t n = 0; n < tgNumOfCols; ++n) { - if (n == priIdx) { - continue; - } - - pColInfo = taosArrayGet(pQueryHandle->pColumns, n); - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; - - assert (pColInfo->info.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX); - - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(pData, pColInfo->info.type); - } else { - setNull(pData, pColInfo->info.type, pColInfo->info.bytes); - } - } - - numOfRows++; - } - - if (numOfRows > 0) { - cur->rows = numOfRows; - cur->mixBlock = true; - - return true; - } - } - - return false; -} - - -static bool loadDataBlockFromTableSeq(STsdbQueryHandle* pQueryHandle) { - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); +//static bool loadCachedLast(STsdbReadHandle* pTsdbReadHandle) { +// // the last row is cached in buffer, return it directly. +// // here note that the pTsdbReadHandle->window must be the TS_INITIALIZER +// int32_t tgNumOfCols = (int32_t)QH_GET_NUM_OF_COLS(pTsdbReadHandle); +// size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); +// int32_t numOfRows = 0; +// assert(numOfTables > 0 && tgNumOfCols > 0); +// SQueryFilePos* cur = &pTsdbReadHandle->cur; +// TSKEY priKey = TSKEY_INITIAL_VAL; +// int32_t priIdx = -1; +// SColumnInfoData* pColInfo = NULL; +// +// while (++pTsdbReadHandle->activeIndex < numOfTables) { +// STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex); +// STable* pTable = pCheckInfo->pTableObj; +// char* pData = NULL; +// +// int32_t numOfCols = pTable->maxColNum; +// +// if (pTable->lastCols == NULL || pTable->maxColNum <= 0) { +// tsdbWarn("no last cached for table %s, uid:%" PRIu64 ",tid:%d", pTable->name->data, pTable->uid, pTable->tableId); +// continue; +// } +// +// int32_t i = 0, j = 0; +// while(i < tgNumOfCols && j < numOfCols) { +// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); +// if (pTable->lastCols[j].colId < pColInfo->info.colId) { +// j++; +// continue; +// } else if (pTable->lastCols[j].colId > pColInfo->info.colId) { +// i++; +// continue; +// } +// +// pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; +// +// if (pTable->lastCols[j].bytes > 0) { +// void* value = pTable->lastCols[j].pData; +// switch (pColInfo->info.type) { +// case TSDB_DATA_TYPE_BINARY: +// case TSDB_DATA_TYPE_NCHAR: +// memcpy(pData, value, varDataTLen(value)); +// break; +// case TSDB_DATA_TYPE_NULL: +// case TSDB_DATA_TYPE_BOOL: +// case TSDB_DATA_TYPE_TINYINT: +// case TSDB_DATA_TYPE_UTINYINT: +// *(uint8_t *)pData = *(uint8_t *)value; +// break; +// case TSDB_DATA_TYPE_SMALLINT: +// case TSDB_DATA_TYPE_USMALLINT: +// *(uint16_t *)pData = *(uint16_t *)value; +// break; +// case TSDB_DATA_TYPE_INT: +// case TSDB_DATA_TYPE_UINT: +// *(uint32_t *)pData = *(uint32_t *)value; +// break; +// case TSDB_DATA_TYPE_BIGINT: +// case TSDB_DATA_TYPE_UBIGINT: +// *(uint64_t *)pData = *(uint64_t *)value; +// break; +// case TSDB_DATA_TYPE_FLOAT: +// SET_FLOAT_PTR(pData, value); +// break; +// case TSDB_DATA_TYPE_DOUBLE: +// SET_DOUBLE_PTR(pData, value); +// break; +// case TSDB_DATA_TYPE_TIMESTAMP: +// if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { +// priKey = tdGetKey(*(TKEY *)value); +// priIdx = i; +// +// i++; +// j++; +// continue; +// } else { +// *(TSKEY *)pData = *(TSKEY *)value; +// } +// break; +// default: +// memcpy(pData, value, pColInfo->info.bytes); +// } +// +// for (int32_t n = 0; n < tgNumOfCols; ++n) { +// if (n == i) { +// continue; +// } +// +// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, n); +// pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; +// +// if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { +//// *(TSKEY *)pData = pTable->lastCols[j].ts; +// continue; +// } +// +// if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { +// setVardataNull(pData, pColInfo->info.type); +// } else { +// setNull(pData, pColInfo->info.type, pColInfo->info.bytes); +// } +// } +// +// numOfRows++; +// assert(numOfRows < pTsdbReadHandle->outputCapacity); +// } +// +// i++; +// j++; +// } +// +// // leave the real ts column as the last row, because last function only (not stable) use the last row as res +// if (priKey != TSKEY_INITIAL_VAL) { +// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, priIdx); +// pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; +// +// *(TSKEY *)pData = priKey; +// +// for (int32_t n = 0; n < tgNumOfCols; ++n) { +// if (n == priIdx) { +// continue; +// } +// +// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, n); +// pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; +// +// assert (pColInfo->info.colId != PRIMARYKEY_TIMESTAMP_COL_ID); +// +// if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { +// setVardataNull(pData, pColInfo->info.type); +// } else { +// setNull(pData, pColInfo->info.type, pColInfo->info.bytes); +// } +// } +// +// numOfRows++; +// } +// +// if (numOfRows > 0) { +// cur->rows = numOfRows; +// cur->mixBlock = true; +// +// return true; +// } +// } +// +// return false; +//} + +static bool loadDataBlockFromTableSeq(STsdbReadHandle* pTsdbReadHandle) { + size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); assert(numOfTables > 0); int64_t stime = taosGetTimestampUs(); - while(pQueryHandle->activeIndex < numOfTables) { - if (loadBlockOfActiveTable(pQueryHandle)) { + while(pTsdbReadHandle->activeIndex < numOfTables) { + if (loadBlockOfActiveTable(pTsdbReadHandle)) { return true; } - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex); pCheckInfo->numOfBlocks = 0; - pQueryHandle->activeIndex += 1; - pQueryHandle->locateStart = false; - pQueryHandle->checkFiles = true; - pQueryHandle->cur.rows = 0; - pQueryHandle->currentLoadExternalRows = pQueryHandle->loadExternalRow; + pTsdbReadHandle->activeIndex += 1; + pTsdbReadHandle->locateStart = false; + pTsdbReadHandle->checkFiles = true; + pTsdbReadHandle->cur.rows = 0; + pTsdbReadHandle->currentLoadExternalRows = pTsdbReadHandle->loadExternalRow; terrno = TSDB_CODE_SUCCESS; int64_t elapsedTime = taosGetTimestampUs() - stime; - pQueryHandle->cost.checkForNextTime += elapsedTime; + pTsdbReadHandle->cost.checkForNextTime += elapsedTime; } return false; } // handle data in cache situation -bool tsdbNextDataBlock(TsdbQueryHandleT pHandle) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; +bool tsdbNextDataBlock(tsdbReadHandleT pHandle) { + STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; - if (emptyQueryTimewindow(pQueryHandle)) { - tsdbDebug("%p query window not overlaps with the data set, no result returned, 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); + if (emptyQueryTimewindow(pTsdbReadHandle)) { + tsdbDebug("%p query window not overlaps with the data set, no result returned, 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId); return false; } @@ -2984,177 +2968,177 @@ bool tsdbNextDataBlock(TsdbQueryHandleT pHandle) { int64_t elapsedTime = stime; // TODO refactor: remove "type" - if (pQueryHandle->type == TSDB_QUERY_TYPE_LAST) { - if (pQueryHandle->cachelastrow == TSDB_CACHED_TYPE_LASTROW) { - return loadCachedLastRow(pQueryHandle); - } else if (pQueryHandle->cachelastrow == TSDB_CACHED_TYPE_LAST) { - return loadCachedLast(pQueryHandle); + if (pTsdbReadHandle->type == TSDB_QUERY_TYPE_LAST) { + if (pTsdbReadHandle->cachelastrow == TSDB_CACHED_TYPE_LASTROW) { +// return loadCachedLastRow(pTsdbReadHandle); + } else if (pTsdbReadHandle->cachelastrow == TSDB_CACHED_TYPE_LAST) { +// return loadCachedLast(pTsdbReadHandle); } } - if (pQueryHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { - return loadDataBlockFromTableSeq(pQueryHandle); + if (pTsdbReadHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { + return loadDataBlockFromTableSeq(pTsdbReadHandle); } else { // loadType == RR and Offset Order - if (pQueryHandle->checkFiles) { + if (pTsdbReadHandle->checkFiles) { // check if the query range overlaps with the file data block bool exists = true; - int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); + int32_t code = getDataBlocksInFiles(pTsdbReadHandle, &exists); if (code != TSDB_CODE_SUCCESS) { - pQueryHandle->activeIndex = 0; - pQueryHandle->checkFiles = false; + pTsdbReadHandle->activeIndex = 0; + pTsdbReadHandle->checkFiles = false; return false; } if (exists) { - pQueryHandle->cost.checkForNextTime += (taosGetTimestampUs() - stime); + pTsdbReadHandle->cost.checkForNextTime += (taosGetTimestampUs() - stime); return exists; } - pQueryHandle->activeIndex = 0; - pQueryHandle->checkFiles = false; + pTsdbReadHandle->activeIndex = 0; + pTsdbReadHandle->checkFiles = false; } // TODO: opt by consider the scan order - bool ret = doHasDataInBuffer(pQueryHandle); + bool ret = doHasDataInBuffer(pTsdbReadHandle); terrno = TSDB_CODE_SUCCESS; elapsedTime = taosGetTimestampUs() - stime; - pQueryHandle->cost.checkForNextTime += elapsedTime; + pTsdbReadHandle->cost.checkForNextTime += elapsedTime; return ret; } } -static int32_t doGetExternalRow(STsdbQueryHandle* pQueryHandle, int16_t type, SMemRef* pMemRef) { - STsdbQueryHandle* pSecQueryHandle = NULL; - - if (type == TSDB_PREV_ROW && pQueryHandle->prev) { - return TSDB_CODE_SUCCESS; - } - - if (type == TSDB_NEXT_ROW && pQueryHandle->next) { - return TSDB_CODE_SUCCESS; - } - - // prepare the structure - int32_t numOfCols = (int32_t) QH_GET_NUM_OF_COLS(pQueryHandle); - - if (type == TSDB_PREV_ROW) { - pQueryHandle->prev = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - if (pQueryHandle->prev == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto out_of_memory; - } - } else { - pQueryHandle->next = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - if (pQueryHandle->next == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto out_of_memory; - } - } - - SArray* row = (type == TSDB_PREV_ROW)? pQueryHandle->prev : pQueryHandle->next; - - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); - - SColumnInfoData colInfo = {{0}, 0}; - colInfo.info = pCol->info; - colInfo.pData = calloc(1, pCol->info.bytes); - if (colInfo.pData == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto out_of_memory; - } - - taosArrayPush(row, &colInfo); - } - - // load the previous row - STsdbQueryCond cond = {.numOfCols = numOfCols, .loadExternalRows = false, .type = BLOCK_LOAD_OFFSET_SEQ_ORDER}; - if (type == TSDB_PREV_ROW) { - cond.order = TSDB_ORDER_DESC; - cond.twindow = (STimeWindow){pQueryHandle->window.skey, INT64_MIN}; - } else { - cond.order = TSDB_ORDER_ASC; - cond.twindow = (STimeWindow){pQueryHandle->window.skey, INT64_MAX}; - } - - cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); - if (cond.colList == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto out_of_memory; - } - - for (int32_t i = 0; i < cond.numOfCols; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, i); - memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo)); - } - - pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qId, pMemRef); - tfree(cond.colList); - - // current table, only one table - STableCheckInfo* pCurrent = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); - - SArray* psTable = NULL; - pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pCurrent, pSecQueryHandle->window.skey, &psTable); - if (pSecQueryHandle->pTableCheckInfo == NULL) { - taosArrayDestroy(psTable); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto out_of_memory; - } - - - tsdbMayTakeMemSnapshot(pSecQueryHandle, psTable); - if (!tsdbNextDataBlock((void*)pSecQueryHandle)) { - // no result in current query, free the corresponding result rows structure - if (type == TSDB_PREV_ROW) { - pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); - } else { - pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); - } - - goto out_of_memory; - } - - SDataBlockInfo blockInfo = {{0}, 0}; - tsdbRetrieveDataBlockInfo((void*)pSecQueryHandle, &blockInfo); - tsdbRetrieveDataBlock((void*)pSecQueryHandle, pSecQueryHandle->defaultLoadColumn); - - row = (type == TSDB_PREV_ROW)? pQueryHandle->prev:pQueryHandle->next; - int32_t pos = (type == TSDB_PREV_ROW)?pSecQueryHandle->cur.rows - 1:0; - - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(row, i); - SColumnInfoData* s = taosArrayGet(pSecQueryHandle->pColumns, i); - memcpy((char*)pCol->pData, (char*)s->pData + s->info.bytes * pos, pCol->info.bytes); - } - -out_of_memory: - tsdbCleanupQueryHandle(pSecQueryHandle); - return terrno; -} - -bool tsdbGetExternalRow(TsdbQueryHandleT pHandle) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; - SQueryFilePos* cur = &pQueryHandle->cur; +//static int32_t doGetExternalRow(STsdbReadHandle* pTsdbReadHandle, int16_t type, STsdbMemTable* pMemRef) { +// STsdbReadHandle* pSecQueryHandle = NULL; +// +// if (type == TSDB_PREV_ROW && pTsdbReadHandle->prev) { +// return TSDB_CODE_SUCCESS; +// } +// +// if (type == TSDB_NEXT_ROW && pTsdbReadHandle->next) { +// return TSDB_CODE_SUCCESS; +// } +// +// // prepare the structure +// int32_t numOfCols = (int32_t) QH_GET_NUM_OF_COLS(pTsdbReadHandle); +// +// if (type == TSDB_PREV_ROW) { +// pTsdbReadHandle->prev = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); +// if (pTsdbReadHandle->prev == NULL) { +// terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto out_of_memory; +// } +// } else { +// pTsdbReadHandle->next = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); +// if (pTsdbReadHandle->next == NULL) { +// terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto out_of_memory; +// } +// } +// +// SArray* row = (type == TSDB_PREV_ROW)? pTsdbReadHandle->prev : pTsdbReadHandle->next; +// +// for (int32_t i = 0; i < numOfCols; ++i) { +// SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pColumns, i); +// +// SColumnInfoData colInfo = {{0}, 0}; +// colInfo.info = pCol->info; +// colInfo.pData = calloc(1, pCol->info.bytes); +// if (colInfo.pData == NULL) { +// terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto out_of_memory; +// } +// +// taosArrayPush(row, &colInfo); +// } +// +// // load the previous row +// STsdbQueryCond cond = {.numOfCols = numOfCols, .loadExternalRows = false, .type = BLOCK_LOAD_OFFSET_SEQ_ORDER}; +// if (type == TSDB_PREV_ROW) { +// cond.order = TSDB_ORDER_DESC; +// cond.twindow = (STimeWindow){pTsdbReadHandle->window.skey, INT64_MIN}; +// } else { +// cond.order = TSDB_ORDER_ASC; +// cond.twindow = (STimeWindow){pTsdbReadHandle->window.skey, INT64_MAX}; +// } +// +// cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); +// if (cond.colList == NULL) { +// terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto out_of_memory; +// } +// +// for (int32_t i = 0; i < cond.numOfCols; ++i) { +// SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, i); +// memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo)); +// } +// +// pSecQueryHandle = tsdbQueryTablesImpl(pTsdbReadHandle->pTsdb, &cond, pTsdbReadHandle->qId, pMemRef); +// tfree(cond.colList); +// +// // current table, only one table +// STableCheckInfo* pCurrent = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex); +// +// SArray* psTable = NULL; +// pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pCurrent, pSecQueryHandle->window.skey, &psTable); +// if (pSecQueryHandle->pTableCheckInfo == NULL) { +// taosArrayDestroy(psTable); +// terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto out_of_memory; +// } +// +// +// tsdbMayTakeMemSnapshot(pSecQueryHandle, psTable); +// if (!tsdbNextDataBlock((void*)pSecQueryHandle)) { +// // no result in current query, free the corresponding result rows structure +// if (type == TSDB_PREV_ROW) { +// pTsdbReadHandle->prev = doFreeColumnInfoData(pTsdbReadHandle->prev); +// } else { +// pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next); +// } +// +// goto out_of_memory; +// } +// +// SDataBlockInfo blockInfo = {{0}, 0}; +// tsdbRetrieveDataBlockInfo((void*)pSecQueryHandle, &blockInfo); +// tsdbRetrieveDataBlock((void*)pSecQueryHandle, pSecQueryHandle->defaultLoadColumn); +// +// row = (type == TSDB_PREV_ROW)? pTsdbReadHandle->prev:pTsdbReadHandle->next; +// int32_t pos = (type == TSDB_PREV_ROW)?pSecQueryHandle->cur.rows - 1:0; +// +// for (int32_t i = 0; i < numOfCols; ++i) { +// SColumnInfoData* pCol = taosArrayGet(row, i); +// SColumnInfoData* s = taosArrayGet(pSecQueryHandle->pColumns, i); +// memcpy((char*)pCol->pData, (char*)s->pData + s->info.bytes * pos, pCol->info.bytes); +// } +// +//out_of_memory: +// tsdbCleanupQueryHandle(pSecQueryHandle); +// return terrno; +//} + +bool tsdbGetExternalRow(tsdbReadHandleT pHandle) { + STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; + SQueryFilePos* cur = &pTsdbReadHandle->cur; cur->fid = INT32_MIN; cur->mixBlock = true; - if (pQueryHandle->prev == NULL || pQueryHandle->next == NULL) { + if (pTsdbReadHandle->prev == NULL || pTsdbReadHandle->next == NULL) { cur->rows = 0; return false; } - int32_t numOfCols = (int32_t) QH_GET_NUM_OF_COLS(pQueryHandle); + int32_t numOfCols = (int32_t) QH_GET_NUM_OF_COLS(pTsdbReadHandle); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, i); - SColumnInfoData* first = taosArrayGet(pQueryHandle->prev, i); + SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* first = taosArrayGet(pTsdbReadHandle->prev, i); memcpy(pColInfoData->pData, first->pData, pColInfoData->info.bytes); - SColumnInfoData* sec = taosArrayGet(pQueryHandle->next, i); + SColumnInfoData* sec = taosArrayGet(pTsdbReadHandle->next, i); memcpy(((char*)pColInfoData->pData) + pColInfoData->info.bytes, sec->pData, pColInfoData->info.bytes); if (i == 0 && pColInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP) { @@ -3171,76 +3155,75 @@ bool tsdbGetExternalRow(TsdbQueryHandleT pHandle) { * if lastRow == NULL, return TSDB_CODE_TDB_NO_CACHE_LAST_ROW * else set pRes and return TSDB_CODE_SUCCESS and save lastKey */ -int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey) { - int32_t code = TSDB_CODE_SUCCESS; - - TSDB_RLOCK_TABLE(pTable); - - if (!pTable->lastRow) { - code = TSDB_CODE_TDB_NO_CACHE_LAST_ROW; - goto out; - } - - if (pRes) { - *pRes = tdMemRowDup(pTable->lastRow); - if (*pRes == NULL) { - code = TSDB_CODE_TDB_OUT_OF_MEMORY; - } - } - -out: - TSDB_RUNLOCK_TABLE(pTable); - return code; -} - -bool isTsdbCacheLastRow(TsdbQueryHandleT* pQueryHandle) { - return ((STsdbQueryHandle *)pQueryHandle)->cachelastrow > TSDB_CACHED_TYPE_NONE; +//int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey) { +// int32_t code = TSDB_CODE_SUCCESS; +// +// TSDB_RLOCK_TABLE(pTable); +// +// if (!pTable->lastRow) { +// code = TSDB_CODE_TDB_NO_CACHE_LAST_ROW; +// goto out; +// } +// +// if (pRes) { +// *pRes = tdMemRowDup(pTable->lastRow); +// if (*pRes == NULL) { +// code = TSDB_CODE_TDB_OUT_OF_MEMORY; +// } +// } +// +//out: +// TSDB_RUNLOCK_TABLE(pTable); +// return code; +//} + +bool isTsdbCacheLastRow(tsdbReadHandleT* pTsdbReadHandle) { + return ((STsdbReadHandle *)pTsdbReadHandle)->cachelastrow > TSDB_CACHED_TYPE_NONE; } -int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList) { - assert(pQueryHandle != NULL && groupList != NULL); - - TSKEY key = TSKEY_INITIAL_VAL; - - SArray* group = taosArrayGetP(groupList->pGroupList, 0); - assert(group != NULL); - - STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(group, 0); - - int32_t code = 0; - - if (((STable*)pInfo->pTable)->lastRow) { - code = tsdbGetCachedLastRow(pInfo->pTable, NULL, &key); - if (code != TSDB_CODE_SUCCESS) { - pQueryHandle->cachelastrow = TSDB_CACHED_TYPE_NONE; - } else { - pQueryHandle->cachelastrow = TSDB_CACHED_TYPE_LASTROW; - } - } - - // update the tsdb query time range - if (pQueryHandle->cachelastrow != TSDB_CACHED_TYPE_NONE) { - pQueryHandle->window = TSWINDOW_INITIALIZER; - pQueryHandle->checkFiles = false; - pQueryHandle->activeIndex = -1; // start from -1 - } +int32_t checkForCachedLastRow(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo *groupList) { + assert(pTsdbReadHandle != NULL && groupList != NULL); + +// TSKEY key = TSKEY_INITIAL_VAL; +// +// SArray* group = taosArrayGetP(groupList->pGroupList, 0); +// assert(group != NULL); +// +// STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(group, 0); +// +// int32_t code = 0; +// +// if (((STable*)pInfo->pTable)->lastRow) { +// code = tsdbGetCachedLastRow(pInfo->pTable, NULL, &key); +// if (code != TSDB_CODE_SUCCESS) { +// pTsdbReadHandle->cachelastrow = TSDB_CACHED_TYPE_NONE; +// } else { +// pTsdbReadHandle->cachelastrow = TSDB_CACHED_TYPE_LASTROW; +// } +// } +// +// // update the tsdb query time range +// if (pTsdbReadHandle->cachelastrow != TSDB_CACHED_TYPE_NONE) { +// pTsdbReadHandle->window = TSWINDOW_INITIALIZER; +// pTsdbReadHandle->checkFiles = false; +// pTsdbReadHandle->activeIndex = -1; // start from -1 +// } - return code; + return TSDB_CODE_SUCCESS; } -int32_t checkForCachedLast(STsdbQueryHandle* pQueryHandle) { - assert(pQueryHandle != NULL); +int32_t checkForCachedLast(STsdbReadHandle* pTsdbReadHandle) { + assert(pTsdbReadHandle != NULL); int32_t code = 0; - - if (pQueryHandle->pTsdb && atomic_load_8(&pQueryHandle->pTsdb->hasCachedLastColumn)){ - pQueryHandle->cachelastrow = TSDB_CACHED_TYPE_LAST; - } +// if (pTsdbReadHandle->pTsdb && atomic_load_8(&pTsdbReadHandle->pTsdb->hasCachedLastColumn)){ +// pTsdbReadHandle->cachelastrow = TSDB_CACHED_TYPE_LAST; +// } // update the tsdb query time range - if (pQueryHandle->cachelastrow) { - pQueryHandle->checkFiles = false; - pQueryHandle->activeIndex = -1; // start from -1 + if (pTsdbReadHandle->cachelastrow) { + pTsdbReadHandle->checkFiles = false; + pTsdbReadHandle->activeIndex = -1; // start from -1 } return code; @@ -3266,7 +3249,7 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { STableKeyInfo* pInfo = (STableKeyInfo*) taosArrayGet(pGroup, i); // if the lastKey equals to INT64_MIN, there is no data in this table - TSKEY lastKey = ((STable*)(pInfo->pTable))->lastKey; + TSKEY lastKey = 0;//((STable*)(pInfo->pTable))->lastKey; if (key < lastKey) { key = lastKey; @@ -3290,7 +3273,7 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { // keyInfo.pTable may be NULL here. if (pInfo->pTable != keyInfo.pTable) { - tsdbUnRefTable(pInfo->pTable); +// tsdbUnRefTable(pInfo->pTable); } } @@ -3322,23 +3305,23 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { return window; } -void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* pDataBlockInfo) { - STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle; +void tsdbRetrieveDataBlockInfo(tsdbReadHandleT* pTsdbReadHandle, SDataBlockInfo* pDataBlockInfo) { + STsdbReadHandle* pHandle = (STsdbReadHandle*)pTsdbReadHandle; SQueryFilePos* cur = &pHandle->cur; - STable* pTable = NULL; + + uint64_t uid = 0; // there are data in file if (pHandle->cur.fid != INT32_MIN) { STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[cur->slot]; - pTable = pBlockInfo->pTableCheckInfo->pTableObj; + uid = pBlockInfo->pTableCheckInfo->tableId; } else { STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - pTable = pCheckInfo->pTableObj; + uid = pCheckInfo->tableId; } - pDataBlockInfo->uid = pTable->tableId.uid; - pDataBlockInfo->tid = pTable->tableId.tid; - pDataBlockInfo->rows = cur->rows; + pDataBlockInfo->uid = uid; + pDataBlockInfo->rows = cur->rows; pDataBlockInfo->window = cur->win; pDataBlockInfo->numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pHandle)); } @@ -3346,8 +3329,8 @@ void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* p /* * return null for mixed data block, if not a complete file data block, the statistics value will always return NULL */ -int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataStatis** pBlockStatis) { - STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; +int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReadHandleT* pTsdbReadHandle, SDataStatis** pBlockStatis) { + STsdbReadHandle* pHandle = (STsdbReadHandle*) pTsdbReadHandle; SQueryFilePos* c = &pHandle->cur; if (c->mixBlock) { @@ -3381,7 +3364,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta // always load the first primary timestamp column data SDataStatis* pPrimaryColStatis = &pHandle->statis[0]; - assert(pPrimaryColStatis->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX); + assert(pPrimaryColStatis->colId == PRIMARYKEY_TIMESTAMP_COL_ID); pPrimaryColStatis->numOfNull = 0; pPrimaryColStatis->min = pBlockInfo->compBlock->keyFirst; @@ -3401,12 +3384,12 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta return TSDB_CODE_SUCCESS; } -SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { +SArray* tsdbRetrieveDataBlock(tsdbReadHandleT* pTsdbReadHandle, SArray* pIdList) { /** * In the following two cases, the data has been loaded to SColumnInfoData. * 1. data is from cache, 2. data block is not completed qualified to query time range */ - STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle; + STsdbReadHandle* pHandle = (STsdbReadHandle*)pTsdbReadHandle; if (pHandle->cur.fid == INT32_MIN) { return pHandle->pColumns; @@ -3417,14 +3400,14 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { if (pHandle->cur.mixBlock) { return pHandle->pColumns; } else { - SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlockInfo->compBlock); + SDataBlockInfo binfo = {0};/*GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlockInfo->compBlock);*/ assert(pHandle->realNumOfRows <= binfo.rows); // data block has been loaded, todo extract method SDataBlockLoadInfo* pBlockLoadInfo = &pHandle->dataBlockLoadInfo; if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fid == pHandle->cur.fid && - pBlockLoadInfo->tid == pCheckInfo->pTableObj->tableId.tid) { + pBlockLoadInfo->uid == pCheckInfo->pTableObj->tid) { return pHandle->pColumns; } else { // only load the file block SBlock* pBlock = pBlockInfo->compBlock; @@ -3451,7 +3434,7 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { } } } - +#if 0 void filterPrepare(void* expr, void* param) { tExprNode* pExpr = (tExprNode*)expr; if (pExpr->_node.info != NULL) { @@ -3550,11 +3533,12 @@ static int32_t tableGroupComparFn(const void *p1, const void *p2, const void *pa return 0; } +#endif static int tsdbCheckInfoCompar(const void* key1, const void* key2) { - if (((STableCheckInfo*)key1)->tableId.tid < ((STableCheckInfo*)key2)->tableId.tid) { + if (((STableCheckInfo*)key1)->tableId < ((STableCheckInfo*)key2)->tableId) { return -1; - } else if (((STableCheckInfo*)key1)->tableId.tid > ((STableCheckInfo*)key2)->tableId.tid) { + } else if (((STableCheckInfo*)key1)->tableId > ((STableCheckInfo*)key2)->tableId) { return 1; } else { ASSERT(false); @@ -3570,7 +3554,6 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable STableKeyInfo info = {.pTable = pTable, .lastKey = skey}; taosArrayPush(g, &info); - tsdbRefTable(pTable); for (int32_t i = 1; i < numOfTables; ++i) { STable** prev = taosArrayGet(pTableList, i - 1); @@ -3579,8 +3562,7 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable int32_t ret = compareFn(prev, p, pSupp); assert(ret == 0 || ret == -1); - tsdbRefTable(*p); - assert((*p)->type == TSDB_CHILD_TABLE); +// assert((*p)->type == TSDB_CHILD_TABLE); if (ret == 0) { STableKeyInfo info1 = {.pTable = *p, .lastKey = skey}; @@ -3597,6 +3579,7 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable taosArrayPush(pGroups, &g); } +#if 0 SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols, TSKEY skey) { assert(pTableList != NULL); SArray* pTableGroup = taosArrayInit(1, POINTER_BYTES); @@ -3616,7 +3599,6 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC for(int32_t i = 0; i < size; ++i) { STableKeyInfo *pKeyInfo = taosArrayGet(pTableList, i); - tsdbRefTable(pKeyInfo->pTable); STableKeyInfo info = {.pTable = pKeyInfo->pTable, .lastKey = skey}; taosArrayPush(sa, &info); @@ -3737,7 +3719,7 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) return TSDB_CODE_SUCCESS; } -int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, +int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo, SColIndex* pColIndex, int32_t numOfCols) { if (tsdbRLockRepoMeta(tsdb) < 0) goto _error; @@ -3752,7 +3734,7 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, cons } if (pTable->type != TSDB_SUPER_TABLE) { - tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", tid:%d, name:%s", tsdb, uid, pTable->tableId.tid, + tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", tid:%d, name:%s", tsdb, uid, pTable->tableId, pTable->name->data); terrno = TSDB_CODE_COM_OPS_NOT_SUPPORT; //basically, this error is caused by invalid sql issued by client @@ -3821,8 +3803,8 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, cons pGroupInfo->numOfTables = (uint32_t)taosArrayGetSize(res); pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, skey); - tsdbDebug("%p stable tid:%d, uid:%"PRIu64" query, numOfTables:%u, belong to %" PRIzu " groups", tsdb, pTable->tableId.tid, - pTable->tableId.uid, pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList)); + tsdbDebug("%p stable tid:%d, uid:%"PRIu64" query, numOfTables:%u, belong to %" PRIzu " groups", tsdb, pTable->tableId, + pTable->uid, pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList)); taosArrayDestroy(res); @@ -3833,7 +3815,7 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, cons return terrno; } -int32_t tsdbGetOneTableGroup(STsdbRepo* tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) { +int32_t tsdbGetOneTableGroup(STsdb* tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) { if (tsdbRLockRepoMeta(tsdb) < 0) goto _error; STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); @@ -3844,7 +3826,6 @@ int32_t tsdbGetOneTableGroup(STsdbRepo* tsdb, uint64_t uid, TSKEY startKey, STab } assert(pTable->type == TSDB_CHILD_TABLE || pTable->type == TSDB_NORMAL_TABLE || pTable->type == TSDB_STREAM_TABLE); - tsdbRefTable(pTable); if (tsdbUnlockRepoMeta(tsdb) < 0) goto _error; pGroupInfo->numOfTables = 1; @@ -3862,7 +3843,7 @@ int32_t tsdbGetOneTableGroup(STsdbRepo* tsdb, uint64_t uid, TSKEY startKey, STab return terrno; } -int32_t tsdbGetTableGroupFromIdList(STsdbRepo* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo) { +int32_t tsdbGetTableGroupFromIdList(STsdb* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo) { if (tsdbRLockRepoMeta(tsdb) < 0) { return terrno; } @@ -3889,8 +3870,6 @@ int32_t tsdbGetTableGroupFromIdList(STsdbRepo* tsdb, SArray* pTableIdList, STabl return terrno; } - tsdbRefTable(pTable); - STableKeyInfo info = {.pTable = pTable, .lastKey = id->key}; taosArrayPush(group, &info); } @@ -3938,42 +3917,42 @@ static void* destroyTableCheckInfo(SArray* pTableCheckInfo) { return NULL; } -void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*)queryHandle; - if (pQueryHandle == NULL) { +void tsdbCleanupQueryHandle(tsdbReadHandleT queryHandle) { + STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)queryHandle; + if (pTsdbReadHandle == NULL) { return; } - pQueryHandle->pColumns = doFreeColumnInfoData(pQueryHandle->pColumns); + pTsdbReadHandle->pColumns = doFreeColumnInfoData(pTsdbReadHandle->pColumns); - taosArrayDestroy(pQueryHandle->defaultLoadColumn); - tfree(pQueryHandle->pDataBlockInfo); - tfree(pQueryHandle->statis); + taosArrayDestroy(pTsdbReadHandle->defaultLoadColumn); + tfree(pTsdbReadHandle->pDataBlockInfo); + tfree(pTsdbReadHandle->statis); - if (!emptyQueryTimewindow(pQueryHandle)) { - tsdbMayUnTakeMemSnapshot(pQueryHandle); + if (!emptyQueryTimewindow(pTsdbReadHandle)) { + tsdbMayUnTakeMemSnapshot(pTsdbReadHandle); } else { - assert(pQueryHandle->pTableCheckInfo == NULL); + assert(pTsdbReadHandle->pTableCheckInfo == NULL); } - if (pQueryHandle->pTableCheckInfo != NULL) { - pQueryHandle->pTableCheckInfo = destroyTableCheckInfo(pQueryHandle->pTableCheckInfo); + if (pTsdbReadHandle->pTableCheckInfo != NULL) { + pTsdbReadHandle->pTableCheckInfo = destroyTableCheckInfo(pTsdbReadHandle->pTableCheckInfo); } - tsdbDestroyReadH(&pQueryHandle->rhelper); + tsdbDestroyReadH(&pTsdbReadHandle->rhelper); - tdFreeDataCols(pQueryHandle->pDataCols); - pQueryHandle->pDataCols = NULL; + tdFreeDataCols(pTsdbReadHandle->pDataCols); + pTsdbReadHandle->pDataCols = NULL; - pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); - pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); + pTsdbReadHandle->prev = doFreeColumnInfoData(pTsdbReadHandle->prev); + pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next); - SIOCostSummary* pCost = &pQueryHandle->cost; + SIOCostSummary* pCost = &pTsdbReadHandle->cost; tsdbDebug("%p :io-cost summary: head-file read cnt:%"PRIu64", head-file time:%"PRIu64" us, statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, 0x%"PRIx64, - pQueryHandle, pCost->headFileLoad, pCost->headFileLoadTime, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qId); + pTsdbReadHandle, pCost->headFileLoad, pCost->headFileLoadTime, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pTsdbReadHandle->qId); - tfree(pQueryHandle); + tfree(pTsdbReadHandle); } void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) { @@ -4257,3 +4236,4 @@ void getTableListfromSkipList(tExprNode *pExpr, SSkipList *pSkipList, SArray *re //apply the hierarchical filter expression to every node in skiplist to find the qualified nodes applyFilterToSkipListNode(pSkipList, pExpr, result, param); } +#endif \ No newline at end of file diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 87a3000d093cecaf97731198bc0df1f3539bae5b..436593f9d6fef03304fd7791787f2fe6e125d564 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -1322,3 +1322,4 @@ int main(int argc, char** argv) { + \ No newline at end of file diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index ba941ab22d58d7dd8755898bad083aa666d15e65..04b5fab4bfd9121c460fd5d5f291d97e98c7ad1e 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -1,12 +1,27 @@ aux_source_directory(src EXECUTOR_SRC) -add_library(executor ${EXECUTOR_SRC}) +#add_library(executor ${EXECUTOR_SRC}) + + +#target_link_libraries( +# executor +# PRIVATE os util common function parser planner qcom tsdb +#) + +add_library(executor STATIC ${EXECUTOR_SRC}) +#set_target_properties(executor PROPERTIES +# IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/libexecutor.a" +# INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/libs/executor" +# ) +target_link_libraries(executor + PRIVATE os util common function parser planner qcom tsdb + ) + target_include_directories( - executor - PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/executor" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" + executor + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/executor" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_link_libraries( - executor - PRIVATE os util common function parser planner qcom -) \ No newline at end of file +#if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +#endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/libs/executor/inc/dataSinkInt.h b/source/libs/executor/inc/dataSinkInt.h index 1bbf5494dd0c102fa533faab4a77f5b48a8462d9..69727626afce01a853c30946bb8f1b9f261c9105 100644 --- a/source/libs/executor/inc/dataSinkInt.h +++ b/source/libs/executor/inc/dataSinkInt.h @@ -31,10 +31,10 @@ typedef struct SDataSinkManager { pthread_mutex_t mutex; } SDataSinkManager; -typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, int32_t* pStatus); -typedef void (*FEndPut)(struct SDataSinkHandle* pHandle); -typedef int32_t (*FGetDataLength)(struct SDataSinkHandle* pHandle, int32_t* pStatus); -typedef int32_t (*FGetDataBlock)(struct SDataSinkHandle* pHandle, SOutPutData* pOutput, int32_t* pStatus); +typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue); +typedef void (*FEndPut)(struct SDataSinkHandle* pHandle, int64_t useconds); +typedef void (*FGetDataLength)(struct SDataSinkHandle* pHandle, int32_t* pLen, bool* pQueryEnd); +typedef int32_t (*FGetDataBlock)(struct SDataSinkHandle* pHandle, SOutputData* pOutput); typedef int32_t (*FDestroyDataSinker)(struct SDataSinkHandle* pHandle); typedef struct SDataSinkHandle { diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 2c1bf71638929ec30b2d25f1d49f6cd7b8da89bd..79766b22ac9caa2676b41fa6cd7fffe67e4a62ce 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -38,7 +38,7 @@ #define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t)) #define GET_RES_EXT_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t) + POINTER_BYTES) -#define GET_QID(_r) (((SQInfo*)((_r)->qinfo))->qId) +#define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.queryId) #define curTimeWindowIndex(_winres) ((_winres)->curIndex) @@ -157,6 +157,6 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, struct STaskRuntimeEnv *pRuntimeEnv, int32_t* offset); -int32_t initUdfInfo(struct SUdfInfo* pUdfInfo); +//int32_t initUdfInfo(struct SUdfInfo* pUdfInfo); #endif // TDENGINE_QUERYUTIL_H diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index ef66b3f2477521b57989ec65dda069e0fe9ef334..59db66beccf6d911616fe5c5c853490bcbb86c34 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -20,6 +20,7 @@ extern "C" { #endif + #ifdef __cplusplus } #endif diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 74ee4637c10eda303c26025cc5da7e88f6e21b7f..d76f2703924ff51ff4cb2d16986c33dbda29a871 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -20,24 +20,19 @@ #include "ttszip.h" #include "tvariant.h" -#include "thash.h" +#include "dataSinkMgt.h" #include "executil.h" +#include "planner.h" #include "taosdef.h" #include "tarray.h" #include "tfilter.h" +#include "thash.h" #include "tlockfree.h" #include "tpagedfile.h" -#include "planner.h" - +#include "executor.h" struct SColumnFilterElem; -typedef struct { - uint32_t numOfTables; - SArray *pGroupList; - SHashObj *map; // speedup acquire the tableQueryInfo by table uid -} STableGroupInfo; - typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); #define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED) @@ -51,19 +46,19 @@ typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int #define NEEDTO_COMPRESS_QUERY(size) ((size) > tsCompressColData? 1 : 0) enum { - // when query starts to execute, this status will set - QUERY_NOT_COMPLETED = 0x1u, + // when this task starts to execute, this status will set + TASK_NOT_COMPLETED = 0x1u, - /* query is over + /* Task is over * 1. this status is used in one row result query process, e.g., count/sum/first/last/ avg...etc. * 2. when all data within queried time window, it is also denoted as query_completed */ - QUERY_COMPLETED = 0x2u, + TASK_COMPLETED = 0x2u, /* when the result is not completed return to client, this status will be * usually used in case of interval query with interpolation option */ - QUERY_OVER = 0x4u, + TASK_OVER = 0x4u, }; typedef struct SResultRowCell { @@ -129,6 +124,7 @@ typedef struct { } SOperatorProfResult; typedef struct STaskCostInfo { + int64_t created; int64_t start; int64_t end; @@ -246,13 +242,14 @@ typedef struct STaskIdInfo { uint64_t taskId; // this is a subplan id } STaskIdInfo; -typedef struct STaskInfo { +typedef struct SExecTaskInfo { STaskIdInfo id; char *content; uint32_t status; STimeWindow window; STaskCostInfo cost; int64_t owner; // if it is in execution + int32_t code; STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure pthread_mutex_t lock; // used to synchronize the rsp/query threads @@ -260,8 +257,10 @@ typedef struct STaskInfo { // int32_t dataReady; // denote if query result is ready or not // void* rspContext; // response context char *sql; // query sql string - jmp_buf env; -} STaskInfo; + jmp_buf env; // + DataSinkHandle dsHandle; + struct SOperatorInfo *pRoot; +} SExecTaskInfo; typedef struct STaskRuntimeEnv { jmp_buf env; @@ -269,7 +268,7 @@ typedef struct STaskRuntimeEnv { uint32_t status; // query status void* qinfo; uint8_t scanFlag; // denotes reversed scan of data or not - void* pQueryHandle; + void* pTsdbReadHandle; int32_t prevGroupId; // previous executed group id bool enableGroupData; @@ -314,8 +313,8 @@ typedef struct SOperatorInfo { char *name; // name, used to show the query execution plan void *info; // extension attribution SExprInfo *pExpr; - STaskRuntimeEnv *pRuntimeEnv; - STaskInfo *pTaskInfo; + STaskRuntimeEnv *pRuntimeEnv; // todo remove it + SExecTaskInfo *pTaskInfo; struct SOperatorInfo **pDownstream; // downstram pointer list int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator @@ -376,7 +375,7 @@ typedef struct STaskParam { } STaskParam; typedef struct STableScanInfo { - void *pQueryHandle; + void *pTsdbReadHandle; int32_t numOfBlocks; int32_t numOfSkipped; int32_t numOfBlockStatis; @@ -544,7 +543,7 @@ typedef struct SOrderOperatorInfo { void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream); SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime); -SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime); +SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); @@ -572,11 +571,11 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pUpstream, int32_t numOfUpstream, SSchema* pSchema, int32_t numOfOutput); SOperatorInfo* createOrderOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrder* pOrderVal); -SSDataBlock* doGlobalAggregate(void* param, bool* newgroup); -SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup); -SSDataBlock* doSLimit(void* param, bool* newgroup); +//SSDataBlock* doGlobalAggregate(void* param, bool* newgroup); +//SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup); +//SSDataBlock* doSLimit(void* param, bool* newgroup); -int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId); +//int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId); void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock); bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p); void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p); @@ -617,14 +616,14 @@ STableQueryInfo* createTmpTableQueryInfo(STimeWindow win); int32_t buildArithmeticExprFromMsg(SExprInfo *pArithExprInfo, void *pQueryMsg); -bool isQueryKilled(SQInfo *pQInfo); +bool isTaskKilled(SExecTaskInfo *pTaskInfo); int32_t checkForQueryBuf(size_t numOfTables); bool checkNeedToCompressQueryCol(SQInfo *pQInfo); bool doBuildResCheck(SQInfo* pQInfo); void setQueryStatus(STaskRuntimeEnv *pRuntimeEnv, int8_t status); bool onlyQueryTags(STaskAttr* pQueryAttr); -void destroyUdfInfo(struct SUdfInfo* pUdfInfo); +//void destroyUdfInfo(struct SUdfInfo* pUdfInfo); bool isValidQInfo(void *param); @@ -644,5 +643,7 @@ void freeQueryAttr(STaskAttr *pQuery); int32_t getMaximumIdleDurationSec(); void doInvokeUdf(struct SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type); +void setTaskStatus(SExecTaskInfo *pTaskInfo, int8_t status); +int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle); #endif // TDENGINE_EXECUTORIMPL_H diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 3d8e51d04d705c502774ddefc84d145a26d7d58c..a69084f3db4966e935ebee2401db7f841781af55 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -19,6 +19,7 @@ #include "tcompression.h" #include "tglobal.h" #include "tqueue.h" +#include "executorimpl.h" #define DATA_META_LENGTH(tables) (sizeof(int32_t) + sizeof(STableIdInfo) * taosHashGetSize(tables) + sizeof(SRetrieveTableRsp)) @@ -42,6 +43,8 @@ typedef struct SDataDispatchHandle { STaosQueue* pDataBlocks; SDataDispatchBuf nextOutput; int32_t status; + bool queryEnd; + int64_t useconds; pthread_mutex_t mutex; } SDataDispatchHandle; @@ -106,10 +109,14 @@ static void toDataCacheEntry(const SDataDispatchHandle* pHandle, const SInputDat SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData; pEntry->compressed = (int8_t)needCompress(pInput->pData, &(pHandle->schema)); pEntry->numOfRows = pInput->pData->info.rows; + pEntry->dataLen = 0; pBuf->useSize = DATA_META_LENGTH(pInput->pTableRetrieveTsMap); copyData(pInput, &pHandle->schema, pEntry->data, pEntry->compressed, &pEntry->dataLen); - pBuf->useSize += (pEntry->compressed ? pEntry->dataLen : pHandle->schema.resultRowSize * pInput->pData->info.rows); + if (0 == pEntry->compressed) { + pEntry->dataLen = pHandle->schema.resultRowSize * pInput->pData->info.rows; + } + pBuf->useSize += pEntry->dataLen; // todo completed } @@ -124,7 +131,9 @@ static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, static int32_t updateStatus(SDataDispatchHandle* pDispatcher) { pthread_mutex_lock(&pDispatcher->mutex); - int32_t status = taosQueueSize(pDispatcher->pDataBlocks) < pDispatcher->pManager->cfg.maxDataBlockNumPerQuery ? DS_CAPACITY_ENOUGH : DS_CAPACITY_FULL; + int32_t blockNums = taosQueueSize(pDispatcher->pDataBlocks); + int32_t status = (0 == blockNums ? DS_BUF_EMPTY : + (blockNums < pDispatcher->pManager->cfg.maxDataBlockNumPerQuery ? DS_BUF_LOW : DS_BUF_FULL)); pDispatcher->status = status; pthread_mutex_unlock(&pDispatcher->mutex); return status; @@ -137,7 +146,7 @@ static int32_t getStatus(SDataDispatchHandle* pDispatcher) { return status; } -static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, int32_t* pStatus) { +static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; SDataDispatchBuf* pBuf = taosAllocateQitem(sizeof(SDataDispatchBuf)); if (NULL == pBuf || !allocBuf(pDispatcher, pInput, pBuf)) { @@ -145,38 +154,46 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, } toDataCacheEntry(pDispatcher, pInput, pBuf); taosWriteQitem(pDispatcher->pDataBlocks, pBuf); - *pStatus = updateStatus(pDispatcher); + *pContinue = (DS_BUF_LOW == updateStatus(pDispatcher) ? true : false); return TSDB_CODE_SUCCESS; } -static void endPut(struct SDataSinkHandle* pHandle) { +static void endPut(struct SDataSinkHandle* pHandle, int64_t useconds) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; pthread_mutex_lock(&pDispatcher->mutex); - pDispatcher->status = DS_END; + pDispatcher->queryEnd = true; + pDispatcher->useconds = useconds; pthread_mutex_unlock(&pDispatcher->mutex); } -static int32_t getDataLength(SDataSinkHandle* pHandle, int32_t* pStatus) { +static void getDataLength(SDataSinkHandle* pHandle, int32_t* pLen, bool* pQueryEnd) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; if (taosQueueEmpty(pDispatcher->pDataBlocks)) { - *pStatus = getStatus(pDispatcher) ? DS_END : DS_IN_PROCESS; - return 0; + *pQueryEnd = pDispatcher->queryEnd; + *pLen = 0; + return; } SDataDispatchBuf* pBuf = NULL; taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); memcpy(&pDispatcher->nextOutput, pBuf, sizeof(SDataDispatchBuf)); taosFreeQitem(pBuf); - return ((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->dataLen; + *pLen = ((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->dataLen; } -static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutPutData* pOutput, int32_t* pStatus) { +static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDispatcher->nextOutput.pData); memcpy(pOutput->pData, pEntry->data, pEntry->dataLen); pOutput->numOfRows = pEntry->numOfRows; pOutput->compressed = pEntry->compressed; tfree(pDispatcher->nextOutput.pData); // todo persistent - *pStatus = updateStatus(pDispatcher); + pOutput->bufStatus = updateStatus(pDispatcher); + pthread_mutex_lock(&pDispatcher->mutex); + pOutput->queryEnd = pDispatcher->queryEnd; + pOutput->needSchedule = false; + pOutput->useconds = pDispatcher->useconds; + pOutput->precision = pDispatcher->schema.precision; + pthread_mutex_unlock(&pDispatcher->mutex); return TSDB_CODE_SUCCESS; } @@ -200,12 +217,14 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSink* pDataS return TSDB_CODE_QRY_OUT_OF_MEMORY; } dispatcher->sink.fPut = putDataBlock; + dispatcher->sink.fEndPut = endPut; dispatcher->sink.fGetLen = getDataLength; dispatcher->sink.fGetData = getDataBlock; dispatcher->sink.fDestroy = destroyDataSinker; dispatcher->pManager = pManager; dispatcher->schema = pDataSink->schema; - dispatcher->status = DS_CAPACITY_ENOUGH; + dispatcher->status = DS_BUF_EMPTY; + dispatcher->queryEnd = false; dispatcher->pDataBlocks = taosOpenQueue(); pthread_mutex_init(&dispatcher->mutex, NULL); if (NULL == dispatcher->pDataBlocks) { diff --git a/source/libs/executor/src/dataSinkMgt.c b/source/libs/executor/src/dataSinkMgt.c index 8a96c5d05fb844f63d646737a5de5c6deb51d24e..80d99f96c68b1d399ebd24b41b505069f0b0947f 100644 --- a/source/libs/executor/src/dataSinkMgt.c +++ b/source/libs/executor/src/dataSinkMgt.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "tarray.h" #include "dataSinkMgt.h" #include "dataSinkInt.h" #include "planner.h" @@ -31,24 +32,24 @@ int32_t dsCreateDataSinker(const struct SDataSink *pDataSink, DataSinkHandle* pH return TSDB_CODE_FAILED; } -int32_t dsPutDataBlock(DataSinkHandle handle, const SInputData* pInput, int32_t* pStatus) { +int32_t dsPutDataBlock(DataSinkHandle handle, const SInputData* pInput, bool* pContinue) { SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle; - return pHandleImpl->fPut(pHandleImpl, pInput, pStatus); + return pHandleImpl->fPut(pHandleImpl, pInput, pContinue); } -void dsEndPut(DataSinkHandle handle) { +void dsEndPut(DataSinkHandle handle, int64_t useconds) { SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle; - return pHandleImpl->fEndPut(pHandleImpl); + return pHandleImpl->fEndPut(pHandleImpl, useconds); } -int32_t dsGetDataLength(DataSinkHandle handle, int32_t* pStatus) { +void dsGetDataLength(DataSinkHandle handle, int32_t* pLen, bool* pQueryEnd) { SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle; - return pHandleImpl->fGetLen(pHandleImpl, pStatus); + pHandleImpl->fGetLen(pHandleImpl, pLen, pQueryEnd); } -int32_t dsGetDataBlock(DataSinkHandle handle, SOutPutData* pOutput, int32_t* pStatus) { +int32_t dsGetDataBlock(DataSinkHandle handle, SOutputData* pOutput) { SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle; - return pHandleImpl->fGetData(pHandleImpl, pOutput, pStatus); + return pHandleImpl->fGetData(pHandleImpl, pOutput); } void dsScheduleProcess(void* ahandle, void* pItem) { diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index e8ecffb72c21a0eee1edbfb4cbfd0e162a59d20b..5253add5afc7241a80f13a48226898b75e8c964c 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -547,7 +547,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv pTableQueryInfoList = malloc(POINTER_BYTES * size); if (pTableQueryInfoList == NULL || posList == NULL || pGroupResInfo->pRows == NULL || pGroupResInfo->pRows == NULL) { -// qError("QInfo:%"PRIu64" failed alloc memory", GET_QID(pRuntimeEnv)); +// qError("QInfo:%"PRIu64" failed alloc memory", GET_TASKID(pRuntimeEnv)); code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _end; } @@ -619,7 +619,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv int64_t endt = taosGetTimestampMs(); -// qDebug("QInfo:%"PRIx64" result merge completed for group:%d, elapsed time:%" PRId64 " ms", GET_QID(pRuntimeEnv), +// qDebug("QInfo:%"PRIx64" result merge completed for group:%d, elapsed time:%" PRId64 " ms", GET_TASKID(pRuntimeEnv), // pGroupResInfo->currentGroup, endt - startt); _end: @@ -641,13 +641,13 @@ int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, STaskRuntimeEnv* pRun break; } -// qDebug("QInfo:%"PRIu64" no result in group %d, continue", GET_QID(pRuntimeEnv), pGroupResInfo->currentGroup); +// qDebug("QInfo:%"PRIu64" no result in group %d, continue", GET_TASKID(pRuntimeEnv), pGroupResInfo->currentGroup); cleanupGroupResInfo(pGroupResInfo); incNextGroup(pGroupResInfo); } // int64_t elapsedTime = taosGetTimestampUs() - st; -// qDebug("QInfo:%"PRIu64" merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", GET_QID(pRuntimeEnv), +// qDebug("QInfo:%"PRIu64" merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", GET_TASKID(pRuntimeEnv), // pGroupResInfo->currentGroup, pGroupResInfo->totalGroup, elapsedTime); return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 968380ea01435f53aaf3e89c949698042478c322..50f69fb567f1f2613f4b05972b20f39e9af38add 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -14,10 +14,12 @@ */ #include "os.h" +#include "tarray.h" +#include "dataSinkMgt.h" +#include "exception.h" #include "tcache.h" #include "tglobal.h" #include "tmsg.h" -#include "exception.h" #include "thash.h" #include "executorimpl.h" @@ -66,152 +68,24 @@ void freeParam(STaskParam *param) { tfree(param->prevResult); } -// todo parse json to get the operator tree. - -int32_t qCreateTask(void* tsdb, int32_t vgId, void* pQueryMsg, qTaskInfo_t* pTaskInfo, uint64_t taskId) { - assert(pQueryMsg != NULL && tsdb != NULL); +int32_t qCreateExecTask(void* tsdb, int32_t vgId, SSubplan* pSubplan, qTaskInfo_t* pTaskInfo) { + assert(tsdb != NULL && pSubplan != NULL); + SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; - int32_t code = TSDB_CODE_SUCCESS; -#if 0 - STaskParam param = {0}; - code = convertQueryMsg(pQueryMsg, ¶m); + int32_t code = doCreateExecTaskInfo(pSubplan, pTask, tsdb); if (code != TSDB_CODE_SUCCESS) { - goto _over; - } - - if (pQueryMsg->numOfTables <= 0) { - qError("Invalid number of tables to query, numOfTables:%d", pQueryMsg->numOfTables); - code = TSDB_CODE_QRY_INVALID_MSG; - goto _over; - } - - if (param.pTableIdList == NULL || taosArrayGetSize(param.pTableIdList) == 0) { - qError("qmsg:%p, SQueryTableReq wrong format", pQueryMsg); - code = TSDB_CODE_QRY_INVALID_MSG; - goto _over; - } - - SQueriedTableInfo info = { .numOfTags = pQueryMsg->numOfTags, .numOfCols = pQueryMsg->numOfCols, .colList = pQueryMsg->tableCols}; - if ((code = createQueryFunc(&info, pQueryMsg->numOfOutput, ¶m.pExprs, param.pExpr, param.pTagColumnInfo, - pQueryMsg->queryType, pQueryMsg, param.pUdfInfo)) != TSDB_CODE_SUCCESS) { - goto _over; - } - - if (param.pSecExpr != NULL) { - if ((code = createIndirectQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->secondStageOutput, ¶m.pSecExprs, param.pSecExpr, param.pExprs, param.pUdfInfo)) != TSDB_CODE_SUCCESS) { - goto _over; - } - } - - if (param.colCond != NULL) { - if ((code = createQueryFilter(param.colCond, pQueryMsg->colCondLen, ¶m.pFilters)) != TSDB_CODE_SUCCESS) { - goto _over; - } - } - - param.pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, param.pGroupColIndex, &code); - if ((param.pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) { - goto _over; - } - - bool isSTableQuery = false; - STableGroupInfo tableGroupInfo = {0}; - int64_t st = taosGetTimestampUs(); - - if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_TABLE_QUERY)) { - STableIdInfo *id = taosArrayGet(param.pTableIdList, 0); - - qDebug("qmsg:%p query normal table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid); - if ((code = tsdbGetOneTableGroup(tsdb, id->uid, pQueryMsg->window.skey, &tableGroupInfo)) != TSDB_CODE_SUCCESS) { - goto _over; - } - } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_STABLE_QUERY)) { - isSTableQuery = true; - - // also note there's possibility that only one table in the super table - if (!TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY)) { - STableIdInfo *id = taosArrayGet(param.pTableIdList, 0); - - // group by normal column, do not pass the group by condition to tsdb to group table into different group - int32_t numOfGroupByCols = pQueryMsg->numOfGroupCols; - if (pQueryMsg->numOfGroupCols == 1 && !TSDB_COL_IS_TAG(param.pGroupColIndex->flag)) { - numOfGroupByCols = 0; - } - - qDebug("qmsg:%p query stable, uid:%"PRIu64", tid:%d", pQueryMsg, id->uid, id->tid); - code = tsdbQuerySTableByTagCond(tsdb, id->uid, pQueryMsg->window.skey, param.tagCond, pQueryMsg->tagCondLen, - pQueryMsg->tagNameRelType, param.tbnameCond, &tableGroupInfo, param.pGroupColIndex, numOfGroupByCols); - - if (code != TSDB_CODE_SUCCESS) { - qError("qmsg:%p failed to query stable, reason: %s", pQueryMsg, tstrerror(code)); - goto _over; - } - } else { - code = tsdbGetTableGroupFromIdList(tsdb, param.pTableIdList, &tableGroupInfo); - if (code != TSDB_CODE_SUCCESS) { - goto _over; - } - - qDebug("qmsg:%p query on %u tables in one group from client", pQueryMsg, tableGroupInfo.numOfTables); - } - - int64_t el = taosGetTimestampUs() - st; - qDebug("qmsg:%p tag filter completed, numOfTables:%u, elapsed time:%"PRId64"us", pQueryMsg, tableGroupInfo.numOfTables, el); - } else { - assert(0); + goto _error; } - code = checkForQueryBuf(tableGroupInfo.numOfTables); - if (code != TSDB_CODE_SUCCESS) { // not enough query buffer, abort - goto _over; - } - - assert(pQueryMsg->stableQuery == isSTableQuery); - (*pTaskInfo) = createQInfoImpl(pQueryMsg, param.pGroupbyExpr, param.pExprs, param.pSecExprs, &tableGroupInfo, - param.pTagColumnInfo, param.pFilters, vgId, param.sql, qId, param.pUdfInfo); - - param.sql = NULL; - param.pExprs = NULL; - param.pSecExprs = NULL; - param.pGroupbyExpr = NULL; - param.pTagColumnInfo = NULL; - param.pFilters = NULL; - - if ((*pTaskInfo) == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _over; - } - param.pUdfInfo = NULL; - - code = initQInfo(&pQueryMsg->tsBuf, tsdb, NULL, *pTaskInfo, ¶m, (char*)pQueryMsg, pQueryMsg->prevResultLen, NULL); - - _over: - if (param.pGroupbyExpr != NULL) { - taosArrayDestroy(param.pGroupbyExpr->columnInfo); - } - - tfree(param.colCond); - - destroyUdfInfo(param.pUdfInfo); - - taosArrayDestroy(param.pTableIdList); - param.pTableIdList = NULL; - - freeParam(¶m); - - for (int32_t i = 0; i < pQueryMsg->numOfCols; i++) { - SColumnInfo* column = pQueryMsg->tableCols + i; - freeColumnFilterInfo(column->flist.filterInfo, column->flist.numOfFilters); - } - - filterFreeInfo(param.pFilters); - - //pTaskInfo already freed in initQInfo, but *pTaskInfo may not pointer to null; + SDataSinkMgtCfg cfg = {.maxDataBlockNum = 1000, .maxDataBlockNumPerQuery = 100}; + code = dsDataSinkMgtInit(&cfg); if (code != TSDB_CODE_SUCCESS) { - *pTaskInfo = NULL; + goto _error; } -#endif + code = dsCreateDataSinker(pSubplan->pDataSink, &(*pTask)->dsHandle); + + _error: // if failed to add ref for all tables in this query, abort current query return code; } @@ -250,7 +124,7 @@ int waitMoment(SQInfo* pQInfo){ while(used_ms < ms) { taosMsleep(1000); used_ms += 1000; - if(isQueryKilled(pQInfo)){ + if(isTaskKilled(pQInfo)){ printf("test check query is canceled, sleep break.%s\n", pQInfo->sql); break; } @@ -261,68 +135,80 @@ int waitMoment(SQInfo* pQInfo){ } #endif -bool qExecTask(qTaskInfo_t qinfo, uint64_t *qId) { - SQInfo *pQInfo = (SQInfo *)qinfo; - assert(pQInfo && pQInfo->signature == pQInfo); - int64_t threadId = taosGetSelfPthreadId(); +int32_t qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + int64_t threadId = taosGetSelfPthreadId(); int64_t curOwner = 0; - if ((curOwner = atomic_val_compare_exchange_64(&pQInfo->owner, 0, threadId)) != 0) { - qError("QInfo:0x%"PRIx64"-%p qhandle is now executed by thread:%p", pQInfo->qId, pQInfo, (void*) curOwner); - pQInfo->code = TSDB_CODE_QRY_IN_EXEC; - return false; + if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) { + qError("QInfo:0x%" PRIx64 "-%p qhandle is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, + (void*)curOwner); + pTaskInfo->code = TSDB_CODE_QRY_IN_EXEC; + return pTaskInfo->code; } - *qId = pQInfo->qId; - if(pQInfo->startExecTs == 0) - pQInfo->startExecTs = taosGetTimestampMs(); - - if (isQueryKilled(pQInfo)) { - qDebug("QInfo:0x%"PRIx64" it is already killed, abort", pQInfo->qId); - return doBuildResCheck(pQInfo); + if (pTaskInfo->cost.start == 0) { + pTaskInfo->cost.start = taosGetTimestampMs(); } - STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - if (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 0) { - qDebug("QInfo:0x%"PRIx64" no table exists for query, abort", pQInfo->qId); -// setTaskStatus(pRuntimeEnv, QUERY_COMPLETED); - return doBuildResCheck(pQInfo); + if (isTaskKilled(pTaskInfo)) { + qDebug("QInfo:0x%" PRIx64 " it is already killed, abort", GET_TASKID(pTaskInfo)); + return pTaskInfo->code; } + // STaskRuntimeEnv* pRuntimeEnv = &pTaskInfo->runtimeEnv; + // if (pTaskInfo->tableqinfoGroupInfo.numOfTables == 0) { + // qDebug("QInfo:0x%"PRIx64" no table exists for query, abort", GET_TASKID(pTaskInfo)); + // setTaskStatus(pTaskInfo, TASK_COMPLETED); + // return doBuildResCheck(pTaskInfo); + // } + // error occurs, record the error code and return to client - int32_t ret = setjmp(pQInfo->runtimeEnv.env); + int32_t ret = setjmp(pTaskInfo->env); if (ret != TSDB_CODE_SUCCESS) { - publishQueryAbortEvent(pQInfo, ret); - pQInfo->code = ret; - qDebug("QInfo:0x%"PRIx64" query abort due to error/cancel occurs, code:%s", pQInfo->qId, tstrerror(pQInfo->code)); - return doBuildResCheck(pQInfo); + publishQueryAbortEvent(pTaskInfo, ret); + pTaskInfo->code = ret; + qDebug("QInfo:0x%" PRIx64 " query abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); + return pTaskInfo->code; } - qDebug("QInfo:0x%"PRIx64" query task is launched", pQInfo->qId); + qDebug("QInfo:0x%" PRIx64 " query task is launched", GET_TASKID(pTaskInfo)); bool newgroup = false; - publishOperatorProfEvent(pRuntimeEnv->proot, QUERY_PROF_BEFORE_OPERATOR_EXEC); + publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_BEFORE_OPERATOR_EXEC); + int64_t st = 0; - int64_t st = taosGetTimestampUs(); - pRuntimeEnv->outputBuf = pRuntimeEnv->proot->exec(pRuntimeEnv->proot, &newgroup); - pQInfo->summary.elapsedTime += (taosGetTimestampUs() - st); -#ifdef TEST_IMPL - waitMoment(pQInfo); -#endif - publishOperatorProfEvent(pRuntimeEnv->proot, QUERY_PROF_AFTER_OPERATOR_EXEC); - pRuntimeEnv->resultInfo.total += GET_NUM_OF_RESULTS(pRuntimeEnv); - - if (isQueryKilled(pQInfo)) { - qDebug("QInfo:0x%"PRIx64" query is killed", pQInfo->qId); - } else if (GET_NUM_OF_RESULTS(pRuntimeEnv) == 0) { - qDebug("QInfo:0x%"PRIx64" over, %u tables queried, total %"PRId64" rows returned", pQInfo->qId, pRuntimeEnv->tableqinfoGroupInfo.numOfTables, - pRuntimeEnv->resultInfo.total); - } else { - qDebug("QInfo:0x%"PRIx64" query paused, %d rows returned, total:%" PRId64 " rows", pQInfo->qId, - GET_NUM_OF_RESULTS(pRuntimeEnv), pRuntimeEnv->resultInfo.total); - } + *handle = pTaskInfo->dsHandle; + + while(1) { + st = taosGetTimestampUs(); + SSDataBlock* pRes = pTaskInfo->pRoot->exec(pTaskInfo->pRoot, &newgroup); - return doBuildResCheck(pQInfo); + pTaskInfo->cost.elapsedTime += (taosGetTimestampUs() - st); + publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pRes == NULL) { // no results generated yet, abort + dsEndPut(pTaskInfo->dsHandle, pTaskInfo->cost.elapsedTime); + return pTaskInfo->code; + } + + bool qcontinue = false; + SInputData inputData = {.pData = pRes, .pTableRetrieveTsMap = NULL}; + pTaskInfo->code = dsPutDataBlock(pTaskInfo->dsHandle, &inputData, &qcontinue); + + if (isTaskKilled(pTaskInfo)) { + qDebug("QInfo:0x%" PRIx64 " task is killed", GET_TASKID(pTaskInfo)); + // } else if (GET_NUM_OF_RESULTS(pRuntimeEnv) == 0) { + // qDebug("QInfo:0x%"PRIx64" over, %u tables queried, total %"PRId64" rows returned", pTaskInfo->qId, pRuntimeEnv->tableqinfoGroupInfo.numOfTables, + // pRuntimeEnv->resultInfo.total); + } + + if (!qcontinue) { + qDebug("QInfo:0x%"PRIx64" query paused, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d", GET_TASKID(pTaskInfo), + 0, 0L, 0); + return pTaskInfo->code; + } + } } int32_t qRetrieveQueryResultInfo(qTaskInfo_t qinfo, bool* buildRes, void* pRspContext) { @@ -398,13 +284,13 @@ int32_t qKillTask(qTaskInfo_t qinfo) { } int32_t qIsTaskCompleted(qTaskInfo_t qinfo) { - SQInfo *pQInfo = (SQInfo *)qinfo; + SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)qinfo; - if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + if (pTaskInfo == NULL /*|| !isValidQInfo(pTaskInfo)*/) { return TSDB_CODE_QRY_INVALID_QHANDLE; } - return isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQInfo->runtimeEnv.status, QUERY_OVER); + return isTaskKilled(pTaskInfo) || Q_STATUS_EQUAL(pTaskInfo->status, TASK_OVER); } void qDestroyTask(qTaskInfo_t qHandle) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b00d37f828161486ba0074e023823dbe1401f220..9aeb97980650775f7f50655e5522f4a88e234c5d 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -26,6 +26,7 @@ #include "thash.h" #include "ttypes.h" #include "query.h" +#include "tsdb.h" #define IS_MAIN_SCAN(runtime) ((runtime)->scanFlag == MAIN_SCAN) #define IS_REVERSE_SCAN(runtime) ((runtime)->scanFlag == REVERSE_SCAN) @@ -204,12 +205,10 @@ static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput); static void destroyAggOperatorInfo(void* param, int32_t numOfOutput); static void destroyOperatorInfo(SOperatorInfo* pOperator); -void setTaskStatus(STaskInfo *pTaskInfo, int8_t status); - static void doSetOperatorCompleted(SOperatorInfo* pOperator) { pOperator->status = OP_EXEC_DONE; if (pOperator->pTaskInfo != NULL) { - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); } } @@ -1509,7 +1508,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn int16_t type = pColInfoData->info.type; if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { - //qError("QInfo:0x%"PRIx64" group by not supported on double/float columns, abort", GET_QID(pRuntimeEnv)); + //qError("QInfo:0x%"PRIx64" group by not supported on double/float columns, abort", GET_TASKID(pRuntimeEnv)); return; } @@ -1908,7 +1907,7 @@ static SQLFunctionCtx* createSQLFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprI pCtx->param[2].i = pQueryAttr->window.ekey; pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; } else if (functionId == FUNCTION_ARITHM) { - pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i); +// pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i); } } @@ -1940,7 +1939,7 @@ static void* destroySQLFunctionCtx(SQLFunctionCtx* pCtx, int32_t numOfOutput) { } static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfTables, SArray* pOperator, void* merger) { - //qDebug("QInfo:0x%"PRIx64" setup runtime env", GET_QID(pRuntimeEnv)); + //qDebug("QInfo:0x%"PRIx64" setup runtime env", GET_TASKID(pRuntimeEnv)); STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; pRuntimeEnv->prevGroupId = INT32_MIN; @@ -1977,7 +1976,7 @@ static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfT } } - //qDebug("QInfo:0x%"PRIx64" init runtime environment completed", GET_QID(pRuntimeEnv)); + //qDebug("QInfo:0x%"PRIx64" init runtime environment completed", GET_TASKID(pRuntimeEnv)); // group by normal column, sliding window query, interval query are handled by interval query processor // interval (down sampling operation) @@ -2164,8 +2163,8 @@ _clean: static void doFreeQueryHandle(STaskRuntimeEnv* pRuntimeEnv) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; -// tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); - pRuntimeEnv->pQueryHandle = NULL; +// tsdbCleanupQueryHandle(pRuntimeEnv->pTsdbReadHandle); + pRuntimeEnv->pTsdbReadHandle = NULL; // SMemRef* pMemRef = &pQueryAttr->memRef; // assert(pMemRef->ref == 0 && pMemRef->snapshot.imem == NULL && pMemRef->snapshot.mem == NULL); @@ -2191,7 +2190,7 @@ static void teardownQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv) { //qDebug("QInfo:0x%"PRIx64" teardown runtime env", pQInfo->qId); destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pQueryAttr->numOfOutput); - destroyUdfInfo(pRuntimeEnv->pUdfInfo); +// destroyUdfInfo(pRuntimeEnv->pUdfInfo); destroyResultBuf(pRuntimeEnv->pResultBuf); doFreeQueryHandle(pRuntimeEnv); @@ -2224,18 +2223,18 @@ static bool needBuildResAfterQueryComplete(SQInfo* pQInfo) { return pQInfo->rspContext != NULL; } -bool isQueryKilled(SQInfo *pQInfo) { - if (IS_QUERY_KILLED(pQInfo)) { +bool isTaskKilled(SExecTaskInfo *pTaskInfo) { + if (IS_QUERY_KILLED(pTaskInfo)) { return true; } // query has been executed more than tsShellActivityTimer, and the retrieve has not arrived // abort current query execution. - if (pQInfo->owner != 0 && ((taosGetTimestampSec() - pQInfo->startExecTs/1000) > getMaximumIdleDurationSec()) && - (!needBuildResAfterQueryComplete(pQInfo))) { + if (pTaskInfo->owner != 0 && ((taosGetTimestampSec() - pTaskInfo->cost.start/1000) > 10*getMaximumIdleDurationSec()) + /*(!needBuildResAfterQueryComplete(pTaskInfo))*/) { - assert(pQInfo->startExecTs != 0); - //qDebug("QInfo:%" PRIu64 " retrieve not arrive beyond %d ms, abort current query execution, start:%" PRId64 + assert(pTaskInfo->cost.start != 0); +// qDebug("QInfo:%" PRIu64 " retrieve not arrive beyond %d ms, abort current query execution, start:%" PRId64 // ", current:%d", pQInfo->qId, 1, pQInfo->startExecTs, taosGetTimestampSec()); return true; } @@ -2887,7 +2886,7 @@ void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFi } } -int32_t loadDataBlock(STaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { +int32_t loadDataBlock(SExecTaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { STaskCostInfo* pCost = &pTaskInfo->cost; pCost->totalBlocks += 1; @@ -2896,13 +2895,15 @@ int32_t loadDataBlock(STaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDa pCost->totalCheckedRows += pBlock->info.rows; pCost->loadBlocks += 1; -// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); + pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); if (pBlock->pDataBlock == NULL) { return terrno; + } else { + return TSDB_CODE_SUCCESS; } } -int32_t loadDataBlockOnDemand(STaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { +int32_t loadDataBlockOnDemand(SExecTaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { *status = BLK_DATA_NO_NEEDED; pBlock->pDataBlock = NULL; @@ -2991,10 +2992,10 @@ int32_t loadDataBlockOnDemand(STaskInfo *pTaskInfo, STableScanInfo* pTableScanIn } else if ((*status) == BLK_DATA_STATIS_NEEDED) { // this function never returns error? pCost->loadBlockStatis += 1; -// tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockAgg); +// tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg); if (pBlock->pBlockAgg == NULL) { // data block statistics does not exist, load data block -// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); +// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); pCost->totalCheckedRows += pBlock->info.rows; } } else { @@ -3002,7 +3003,7 @@ int32_t loadDataBlockOnDemand(STaskInfo *pTaskInfo, STableScanInfo* pTableScanIn // load the data block statistics to perform further filter pCost->loadBlockStatis += 1; -// tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockAgg); +// tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg); if (pQueryAttr->topBotQuery && pBlock->pBlockAgg != NULL) { { // set previous window @@ -3048,7 +3049,7 @@ int32_t loadDataBlockOnDemand(STaskInfo *pTaskInfo, STableScanInfo* pTableScanIn pCost->totalCheckedRows += pBlockInfo->rows; pCost->loadBlocks += 1; -// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); +// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); // if (pBlock->pDataBlock == NULL) { // return terrno; // } @@ -3449,12 +3450,12 @@ void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size) { } } -void setTaskStatus(STaskInfo *pTaskInfo, int8_t status) { - if (status == QUERY_NOT_COMPLETED) { +void setTaskStatus(SExecTaskInfo *pTaskInfo, int8_t status) { + if (status == TASK_NOT_COMPLETED) { pTaskInfo->status = status; } else { // QUERY_NOT_COMPLETED is not compatible with any other status, so clear its position first - CLEAR_QUERY_STATUS(pTaskInfo, QUERY_NOT_COMPLETED); + CLEAR_QUERY_STATUS(pTaskInfo, TASK_NOT_COMPLETED); pTaskInfo->status |= status; } } @@ -3476,6 +3477,10 @@ static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SQLFunctionCt SWITCH_ORDER(pTableScanInfo->order); setupQueryRangeForReverseScan(pTableScanInfo); + + pTableScanInfo->times = 1; + pTableScanInfo->current = 0; + pTableScanInfo->reverseTimes = 0; } void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { @@ -3700,10 +3705,10 @@ void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprI // // int16_t tagType = pCtx[0].tag.nType; // if (tagType == TSDB_DATA_TYPE_BINARY || tagType == TSDB_DATA_TYPE_NCHAR) { -// //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%s", GET_QID(pRuntimeEnv), +// //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%s", GET_TASKID(pRuntimeEnv), //// pExprInfo->base.param[0].i, pCtx[0].tag.pz); // } else { -// //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64, GET_QID(pRuntimeEnv), +// //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64, GET_TASKID(pRuntimeEnv), //// pExprInfo->base.param[0].i, pCtx[0].tag.i); // } // } @@ -3723,9 +3728,9 @@ int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, S // failed to find data with the specified tag value and vnodeId if (!tsBufIsValidElem(&elem)) { if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - //qError("QInfo:0x%"PRIx64" failed to find tag:%s in ts_comp", GET_QID(pRuntimeEnv), pTag->pz); + //qError("QInfo:0x%"PRIx64" failed to find tag:%s in ts_comp", GET_TASKID(pRuntimeEnv), pTag->pz); } else { - //qError("QInfo:0x%"PRIx64" failed to find tag:%" PRId64 " in ts_comp", GET_QID(pRuntimeEnv), pTag->i); + //qError("QInfo:0x%"PRIx64" failed to find tag:%" PRId64 " in ts_comp", GET_TASKID(pRuntimeEnv), pTag->i); } return -1; @@ -3734,17 +3739,17 @@ int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, S // Keep the cursor info of current table pTableQueryInfo->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - //qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + //qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } else { - //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } } else { tsBufSetCursor(pRuntimeEnv->pTsBuf, &pTableQueryInfo->cur); if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - //qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + //qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } else { - //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } } @@ -3882,7 +3887,7 @@ static int32_t doCopyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, SGroupResInfo* p int32_t start = 0; int32_t step = -1; - //qDebug("QInfo:0x%"PRIx64" start to copy data from windowResInfo to output buf", GET_QID(pRuntimeEnv)); + //qDebug("QInfo:0x%"PRIx64" start to copy data from windowResInfo to output buf", GET_TASKID(pRuntimeEnv)); assert(orderType == TSDB_ORDER_ASC || orderType == TSDB_ORDER_DESC); if (orderType == TSDB_ORDER_ASC) { @@ -3927,7 +3932,7 @@ static int32_t doCopyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, SGroupResInfo* p } } - //qDebug("QInfo:0x%"PRIx64" copy data to query buf completed", GET_QID(pRuntimeEnv)); + //qDebug("QInfo:0x%"PRIx64" copy data to query buf completed", GET_TASKID(pRuntimeEnv)); pBlock->info.rows = numOfResult; return 0; } @@ -4055,7 +4060,7 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data //qDebug("QInfo:0x%"PRIx64" set %d subscribe info", pQInfo->qId, total); // Check if query is completed or not for stable query or normal table query respectively. - if (Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED) && pRuntimeEnv->proot->status == OP_EXEC_DONE) { + if (Q_STATUS_EQUAL(pRuntimeEnv->status, TASK_COMPLETED) && pRuntimeEnv->proot->status == OP_EXEC_DONE) { // setTaskStatus(pOperator->pTaskInfo, QUERY_OVER); } } @@ -4232,7 +4237,7 @@ void queryCostStatis(SQInfo *pQInfo) { // // assert(pQueryAttr->pos >= 0 && pQueryAttr->pos <= pBlockInfo->rows - 1); // -// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // // // update the pQueryAttr->limit.offset value, and pQueryAttr->pos value @@ -4244,7 +4249,7 @@ void queryCostStatis(SQInfo *pQInfo) { // // int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); // -// //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numBlocksOfStep:%d, numOfRes:%d, lastKey:%"PRId64, GET_QID(pRuntimeEnv), +// //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numBlocksOfStep:%d, numOfRes:%d, lastKey:%"PRId64, GET_TASKID(pRuntimeEnv), // pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, pQuery->current->lastKey); //} @@ -4259,22 +4264,22 @@ void queryCostStatis(SQInfo *pQInfo) { // int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); // // STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; -// TsdbQueryHandleT pQueryHandle = pRuntimeEnv->pQueryHandle; +// TsdbQueryHandleT pTsdbReadHandle = pRuntimeEnv->pTsdbReadHandle; // // SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; -// while (tsdbNextDataBlock(pQueryHandle)) { -// if (isQueryKilled(pRuntimeEnv->qinfo)) { +// while (tsdbNextDataBlock(pTsdbReadHandle)) { +// if (isTaskKilled(pRuntimeEnv->qinfo)) { // longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); // } // -// tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); +// tsdbRetrieveDataBlockInfo(pTsdbReadHandle, &blockInfo); // // if (pQueryAttr->limit.offset > blockInfo.rows) { // pQueryAttr->limit.offset -= blockInfo.rows; // pTableQueryInfo->lastKey = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? blockInfo.window.ekey : blockInfo.window.skey; // pTableQueryInfo->lastKey += step; // -// //qDebug("QInfo:0x%"PRIx64" skip rows:%d, offset:%" PRId64, GET_QID(pRuntimeEnv), blockInfo.rows, +// //qDebug("QInfo:0x%"PRIx64" skip rows:%d, offset:%" PRId64, GET_TASKID(pRuntimeEnv), blockInfo.rows, // pQuery->limit.offset); // } else { // find the appropriated start position in current block // updateOffsetVal(pRuntimeEnv, &blockInfo); @@ -4300,7 +4305,7 @@ void queryCostStatis(SQInfo *pQInfo) { // // // load the data block and check data remaining in current data block // // TODO optimize performance -// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // // tw = *win; @@ -4323,7 +4328,7 @@ void queryCostStatis(SQInfo *pQInfo) { // pRuntimeEnv->resultRowInfo.curIndex = index; // restore the window index // // //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%" PRId64, -// GET_QID(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, +// GET_TASKID(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, // pQueryAttr->current->lastKey); // // return key; @@ -4365,8 +4370,8 @@ void queryCostStatis(SQInfo *pQInfo) { // STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; // // SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; -// while (tsdbNextDataBlock(pRuntimeEnv->pQueryHandle)) { -// tsdbRetrieveDataBlockInfo(pRuntimeEnv->pQueryHandle, &blockInfo); +// while (tsdbNextDataBlock(pRuntimeEnv->pTsdbReadHandle)) { +// tsdbRetrieveDataBlockInfo(pRuntimeEnv->pTsdbReadHandle, &blockInfo); // // if (QUERY_IS_ASC_QUERY(pQueryAttr)) { // if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { @@ -4412,7 +4417,7 @@ void queryCostStatis(SQInfo *pQInfo) { // */ // if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) { // -// SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // // if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) { @@ -4486,7 +4491,7 @@ static int32_t setupQueryHandle(void* tsdb, STaskRuntimeEnv* pRuntimeEnv, int64_ terrno = TSDB_CODE_SUCCESS; if (isFirstLastRowQuery(pQueryAttr)) { - pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryLastRow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); // update the query time window pQueryAttr->window = cond.twindow; @@ -4507,11 +4512,11 @@ static int32_t setupQueryHandle(void* tsdb, STaskRuntimeEnv* pRuntimeEnv, int64_ } } } else if (isCachedLastQuery(pQueryAttr)) { - pRuntimeEnv->pQueryHandle = tsdbQueryCacheLast(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryCacheLast(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); } else if (pQueryAttr->pointInterpQuery) { - pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); } else { - pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryTables(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); } #endif return terrno; @@ -4543,19 +4548,19 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr switch(tbScanner) { // case OP_TableBlockInfoScan: { -// pRuntimeEnv->proot = createTableBlockInfoScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); +// pRuntimeEnv->proot = createTableBlockInfoScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv); // break; // } // case OP_TableSeqScan: { -// pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); +// pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv); // break; // } // case OP_DataBlocksOptScan: { -// pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0); +// pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0); // break; // } // case OP_TableScan: { -// pRuntimeEnv->proot = createTableScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr)); +// pRuntimeEnv->proot = createTableScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr)); // break; // } default: { // do nothing @@ -4606,7 +4611,7 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr return TSDB_CODE_SUCCESS; } -static void doTableQueryInfoTimeWindowCheck(STaskInfo* pTaskInfo, STableQueryInfo* pTableQueryInfo, int32_t order) { +static void doTableQueryInfoTimeWindowCheck(SExecTaskInfo* pTaskInfo, STableQueryInfo* pTableQueryInfo, int32_t order) { if (order == TSDB_ORDER_ASC) { assert( (pTableQueryInfo->win.skey <= pTableQueryInfo->win.ekey) && @@ -4679,20 +4684,20 @@ static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { SOperatorInfo *pOperator = (SOperatorInfo*) param; STableScanInfo *pTableScanInfo = pOperator->info; - STaskInfo *pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; SSDataBlock *pBlock = &pTableScanInfo->block; STableGroupInfo *pTableGroupInfo = &pOperator->pTaskInfo->tableqinfoGroupInfo; *newgroup = false; - while (/*tsdbNextDataBlock(pTableScanInfo->pQueryHandle)*/1) { - if (isQueryKilled(pOperator->pRuntimeEnv->qinfo)) { - longjmp(pOperator->pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); + while (tsdbNextDataBlock(pTableScanInfo->pTsdbReadHandle)) { + if (isTaskKilled(pOperator->pTaskInfo)) { + longjmp(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); } pTableScanInfo->numOfBlocks += 1; -// tsdbRetrieveDataBlockInfo(pTableScanInfo->pQueryHandle, &pBlock->info); + tsdbRetrieveDataBlockInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->info); // todo opt // if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) { @@ -4711,7 +4716,7 @@ static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { int32_t code = loadDataBlock(pTaskInfo, pTableScanInfo, pBlock, &status); // int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status); if (code != TSDB_CODE_SUCCESS) { - longjmp(pOperator->pRuntimeEnv->env, code); + longjmp(pOperator->pTaskInfo->env, code); } // current block is ignored according to filter result by block statistics data, continue load the next block @@ -4729,7 +4734,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { SOperatorInfo* pOperator = (SOperatorInfo*) param; STableScanInfo *pTableScanInfo = pOperator->info; - STaskInfo *pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; SResultRowInfo* pResultRowInfo = pTableScanInfo->pResultRowInfo; *newgroup = false; @@ -4741,7 +4746,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { } if (++pTableScanInfo->current >= pTableScanInfo->times) { - if (pTableScanInfo->reverseTimes <= 0/* || isTsdbCacheLastRow(pTableScanInfo->pQueryHandle)*/) { + if (pTableScanInfo->reverseTimes <= 0/* || isTsdbCacheLastRow(pTableScanInfo->pTsdbReadHandle)*/) { return NULL; } else { break; @@ -4750,9 +4755,9 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { // do prepare for the next round table scan operation // STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); -// tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); +// tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond); - setTaskStatus(pTaskInfo, QUERY_NOT_COMPLETED); + setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTableScanInfo->scanFlag = REPEAT_SCAN; // if (pTaskInfo->pTsBuf) { @@ -4764,8 +4769,8 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { pResultRowInfo->curPos = 0; } - //qDebug("QInfo:0x%"PRIx64" start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, -// GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey); + qDebug("QInfo:0x%"PRIx64" start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, + GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); } SSDataBlock *p = NULL; @@ -4773,15 +4778,10 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { if (pTableScanInfo->reverseTimes > 0) { setupEnvForReverseScan(pTableScanInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); // STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); -// tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); - - //qDebug("QInfo:0x%"PRIx64" start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, -// GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey); +// tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond); - pTableScanInfo->times = 1; - pTableScanInfo->current = 0; - pTableScanInfo->reverseTimes = 0; -// pTableScanInfo->order = cond.order; + qDebug("QInfo:0x%"PRIx64" start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, + GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); if (pResultRowInfo->size > 0) { pResultRowInfo->curPos = pResultRowInfo->size - 1; @@ -4814,8 +4814,8 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { tableBlockDist.maxRows = INT_MIN; tableBlockDist.minRows = INT_MAX; - tsdbGetFileBlocksDistInfo(pTableScanInfo->pQueryHandle, &tableBlockDist); - tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pQueryHandle); + tsdbGetFileBlocksDistInfo(pTableScanInfo->pTsdbReadHandle, &tableBlockDist); + tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pTsdbReadHandle); SSDataBlock* pBlock = &pTableScanInfo->block; pBlock->info.rows = 1; @@ -4842,11 +4842,11 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { } -SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime) { +SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo) { assert(repeatTime > 0 && numOfOutput > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->times = repeatTime; pInfo->reverseTimes = 0; pInfo->order = order; @@ -4862,6 +4862,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, in pOperator->numOfOutput = numOfOutput; pOperator->pRuntimeEnv = NULL; pOperator->exec = doTableScan; + pOperator->pTaskInfo = pTaskInfo; return pOperator; } @@ -4869,7 +4870,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, in SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv) { STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->times = 1; pInfo->reverseTimes = 0; pInfo->order = pRuntimeEnv->pQueryAttr->order.order; @@ -4893,7 +4894,7 @@ SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEn SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv) { STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); SColumnInfoData infoData = {{0}}; @@ -4977,7 +4978,7 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, STaskRuntimeE assert(repeatTime > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->times = repeatTime; pInfo->reverseTimes = reverseTime; pInfo->current = 0; @@ -5145,7 +5146,7 @@ SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S pOperator->numOfOutput = numOfOutput; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doGlobalAggregate; +// pOperator->exec = doGlobalAggregate; pOperator->cleanup = destroyGlobalAggOperatorInfo; appendUpstream(pOperator, downstream); @@ -5188,7 +5189,7 @@ SOperatorInfo *createMultiwaySortOperatorInfo(STaskRuntimeEnv *pRuntimeEnv, SExp pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->numOfOutput = numOfOutput; pOperator->pExpr = pExpr; - pOperator->exec = doMultiwayMergeSort; +// pOperator->exec = doMultiwayMergeSort; pOperator->cleanup = destroyGlobalAggOperatorInfo; return pOperator; } @@ -5476,7 +5477,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { assert(*newgroup == false); *newgroup = prevVal; - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); break; } @@ -5644,7 +5645,7 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pIntervalInfo->resultRowInfo); - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); @@ -5704,7 +5705,7 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pIntervalInfo->resultRowInfo); - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); @@ -5767,7 +5768,7 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; pQueryAttr->order.order = order; // TODO : restore the order doCloseAllTimeWindow(pRuntimeEnv); - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { @@ -5822,7 +5823,7 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; pQueryAttr->order.order = order; // TODO : restore the order doCloseAllTimeWindow(pRuntimeEnv); - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); int64_t st = taosGetTimestampUs(); copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); @@ -5956,7 +5957,7 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pBInfo->resultRowInfo); - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); finalizeQueryResult(pOperator, pBInfo->pCtx, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo); @@ -6094,7 +6095,7 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo *pInfo, STaskRuntimeEnv* pRuntimeEnv, bool* newgroup) { pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows; - int64_t ekey = Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED)?pRuntimeEnv->pQueryAttr->window.ekey:pInfo->existNewGroupBlock->info.window.ekey; + int64_t ekey = Q_STATUS_EQUAL(pRuntimeEnv->status, TASK_COMPLETED)?pRuntimeEnv->pQueryAttr->window.ekey:pInfo->existNewGroupBlock->info.window.ekey; taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo)); taosFillSetStartInfo(pInfo->pFillInfo, pInfo->existNewGroupBlock->info.rows, ekey); @@ -6422,7 +6423,7 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI SFilterOperatorInfo* pInfo = calloc(1, sizeof(SFilterOperatorInfo)); assert(numOfFilter > 0 && pCols != NULL); - doCreateFilterInfo(pCols, numOfOutput, numOfFilter, &pInfo->pFilterInfo, 0); +// doCreateFilterInfo(pCols, numOfOutput, numOfFilter, &pInfo->pFilterInfo, 0); pInfo->numOfFilterCols = numOfFilter; SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); @@ -6714,10 +6715,10 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "SLimitOperator"; -// pOperator->operatorType = OP_SLimit; + pOperator->operatorType = OP_SLimit; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; - pOperator->exec = doSLimit; +// pOperator->exec = doSLimit; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->cleanup = destroySlimitOperatorInfo; @@ -6799,14 +6800,14 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { count += 1; } - //qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_QID(pRuntimeEnv), count); + //qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_TASKID(pRuntimeEnv), count); } else if (functionId == FUNCTION_COUNT) {// handle the "count(tbname)" query SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); *(int64_t*)pColInfo->pData = pInfo->totalTables; count = 1; pOperator->status = OP_EXEC_DONE; - //qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_QID(pRuntimeEnv), count); + //qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_TASKID(pRuntimeEnv), count); } else { // return only the tags|table name etc. SExprInfo* pExprInfo = &pOperator->pExpr[0]; // todo use the column list instead of exprinfo @@ -6845,11 +6846,11 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { pOperator->status = OP_EXEC_DONE; } - //qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_QID(pRuntimeEnv), count); + //qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count); } if (pOperator->status == OP_EXEC_DONE) { - setTaskStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); + setTaskStatus(pOperator->pRuntimeEnv, TASK_COMPLETED); } pRes->info.rows = count; @@ -7174,90 +7175,53 @@ static int32_t deserializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t return TSDB_CODE_SUCCESS; } -/** - * { - "Id": { - "QueryId": 20, - "TemplateId": 0, - "SubplanId": 0 - }, - "Node": { - "Name": "TableScan", - "InputSchema": [{ - "Type": 9, - "ColId": 1, - "Bytes": 8 - }, { - "Type": 4, - "ColId": 2, - "Bytes": 4 - }, { - "Type": 8, - "ColId": 3, - "Bytes": 20 - }], - "TableScan": { - "TableId": 1, - "TableType": 3, - "Flag": 0, - "Window": { - "StartKey": 0, - "EndKey": 0 - } - } - }, - "DataSink": { - "Name": "Dispatch", - "Dispatch": { - } - } -} - */ -int32_t parseTaskInfo(const char* msg, int32_t len) { - cJSON* pJson = cJSON_Parse(msg); - if (NULL == pJson) { - return TSDB_CODE_INVALID_MSG; - } - - cJSON* pSub = cJSON_GetObjectItem(pJson, "ID"); - if (NULL != pSub) { - printf("Id : %s\n", pSub->valuestring); - } +static SExecTaskInfo* createExecTaskInfo(uint64_t queryId) { + SExecTaskInfo* pTaskInfo = calloc(1, sizeof(SExecTaskInfo)); + setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); - cJSON* pNode = cJSON_GetObjectItem(pJson, "Node"); - if (pNode == NULL) { - return TSDB_CODE_INVALID_MSG; - } + pthread_mutex_init(&pTaskInfo->lock, NULL); + pTaskInfo->cost.created = taosGetTimestampMs(); + pTaskInfo->id.queryId = queryId; + return pTaskInfo; +} - cJSON* pNodeName = cJSON_GetObjectItem(pNode, "name"); - if (pNodeName == NULL) { - return TSDB_CODE_INVALID_MSG; +SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTaskInfo, void* param) { + if (pPhyNode->pChildren == NULL || taosArrayGetSize(pPhyNode->pChildren) == 0) { + if (pPhyNode->info.type == OP_TableScan) { + size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); + SOperatorInfo* pOperatorInfo = createTableScanOperator(param, TSDB_ORDER_ASC, numOfCols, 1, pTaskInfo); + pTaskInfo->pRoot = pOperatorInfo; + } } +} - printf("node name is: %s\n", pNodeName->valuestring); +int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle) { + STsdbQueryCond cond = {.order = TSDB_ORDER_ASC, .numOfCols = 2, .loadExternalRows = false}; + cond.twindow.skey = INT64_MIN; + cond.twindow.ekey = INT64_MAX; + cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); - cJSON* pNodeSchema = cJSON_GetObjectItem(pNode, "InputSchema"); - if (pNodeSchema == NULL) { - return TSDB_CODE_INVALID_MSG; - } + // todo set the correct table column info + cond.colList[0].type = TSDB_DATA_TYPE_TIMESTAMP; + cond.colList[0].bytes = sizeof(uint64_t); + cond.colList[0].colId = 1; - cJSON* pOperator = cJSON_GetObjectItem(pNode, pNodeName->valuestring); - if (pOperator == NULL) { - return TSDB_CODE_INVALID_MSG; - } + cond.colList[1].type = TSDB_DATA_TYPE_INT; + cond.colList[1].bytes = sizeof(int32_t); + cond.colList[1].colId = 2; - cJSON* pTableId = cJSON_GetObjectItem(pOperator, "tableId"); - if (pTableId == NULL) { - return TSDB_CODE_INVALID_MSG; - } - - cJSON* pTimeWindow = cJSON_GetObjectItem(pOperator, "window"); - if (pTimeWindow == NULL) { - return TSDB_CODE_INVALID_MSG; - } + STableGroupInfo group = {.numOfTables = 1, .pGroupList = taosArrayInit(1, POINTER_BYTES)}; + SArray* pa = taosArrayInit(1, sizeof(STableKeyInfo)); + STableKeyInfo info = {.pTable = NULL, .lastKey = 0, .uid = 1}; + taosArrayPush(pa, &info); + taosArrayPush(group.pGroupList, &pa); + *pTaskInfo = createExecTaskInfo((uint64_t)pPlan->id.queryId); + tsdbReadHandleT tsdbReadHandle = tsdbQueryTables(readerHandle, &cond, &group, (*pTaskInfo)->id.queryId, NULL); + doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, tsdbReadHandle); + return TSDB_CODE_SUCCESS; } /** @@ -7704,7 +7668,7 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp *pExprInfo = NULL; int32_t code = TSDB_CODE_SUCCESS; - code = initUdfInfo(pUdfInfo); +// code = initUdfInfo(pUdfInfo); if (code) { return code; } @@ -7996,8 +7960,8 @@ int32_t createFilterInfo(STaskAttr* pQueryAttr, uint64_t qId) { return TSDB_CODE_SUCCESS; } - doCreateFilterInfo(pQueryAttr->tableCols, pQueryAttr->numOfCols, pQueryAttr->numOfFilterCols, - &pQueryAttr->pFilterInfo, qId); +// doCreateFilterInfo(pQueryAttr->tableCols, pQueryAttr->numOfCols, pQueryAttr->numOfFilterCols, +// &pQueryAttr->pFilterInfo, qId); pQueryAttr->createFilterOperator = true; @@ -8505,7 +8469,7 @@ int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t } // all data returned, set query over - if (Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED)) { + if (Q_STATUS_EQUAL(pRuntimeEnv->status, TASK_COMPLETED)) { // setTaskStatus(pOperator->pTaskInfo, QUERY_OVER); } } else { diff --git a/source/libs/executor/test/CMakeLists.txt b/source/libs/executor/test/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..ece84207c7912b8e0c45054a691596ecf3ab0bb5 --- /dev/null +++ b/source/libs/executor/test/CMakeLists.txt @@ -0,0 +1,18 @@ + +MESSAGE(STATUS "build parser unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(executorTest ${SOURCE_LIST}) +TARGET_LINK_LIBRARIES( + executorTest + PUBLIC os util common transport gtest taos qcom executor function planner +) + +TARGET_INCLUDE_DIRECTORIES( + executorTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/executor/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/executor/inc" +) diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..5f5fddbe28ab3106eb1e7caedad4f3a88ca19bef 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#pragma GCC diagnostic ignored "-Wwrite-strings" + +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#include "os.h" + +#include "taos.h" +#include "tdef.h" +#include "tvariant.h" +#include "tep.h" +#include "trpc.h" +#include "stub.h" +#include "executor.h" + +/** +{ + "Id": { + "QueryId": 1.3108161807422521e+19, + "TemplateId": 0, + "SubplanId": 0 + }, + "Node": { + "Name": "TableScan", + "Targets": [{ + "Base": { + "Schema": { + "Type": 9, + "ColId": 5000, + "Bytes": 8 + }, + "Columns": [{ + "TableId": 1, + "Flag": 0, + "Info": { + "ColId": 1, + "Type": 9, + "Bytes": 8 + } + }], + "InterBytes": 0 + }, + "Expr": { + "Type": 4, + "Column": { + "Type": 9, + "ColId": 1, + "Bytes": 8 + } + } + }, { + "Base": { + "Schema": { + "Type": 4, + "ColId": 5001, + "Bytes": 4 + }, + "Columns": [{ + "TableId": 1, + "Flag": 0, + "Info": { + "ColId": 2, + "Type": 4, + "Bytes": 4 + } + }], + "InterBytes": 0 + }, + "Expr": { + "Type": 4, + "Column": { + "Type": 4, + "ColId": 2, + "Bytes": 4 + } + } + }], + "InputSchema": [{ + "Type": 9, + "ColId": 5000, + "Bytes": 8 + }, { + "Type": 4, + "ColId": 5001, + "Bytes": 4 + }], + "TableScan": { + "TableId": 1, + "TableType": 2, + "Flag": 0, + "Window": { + "StartKey": -9.2233720368547758e+18, + "EndKey": 9.2233720368547758e+18 + } + } + }, + "DataSink": { + "Name": "Dispatch", + "Dispatch": { + } + } +} + */ + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(testCase, build_executor_tree_Test) { + + + const char* msg = "{\n" + "\t\"Id\":\t{\n" + "\t\t\"QueryId\":\t1.3108161807422521e+19,\n" + "\t\t\"TemplateId\":\t0,\n" + "\t\t\"SubplanId\":\t0\n" + "\t},\n" + "\t\"Node\":\t{\n" + "\t\t\"Name\":\t\"TableScan\",\n" + "\t\t\"Targets\":\t[{\n" + "\t\t\t\t\"Base\":\t{\n" + "\t\t\t\t\t\"Schema\":\t{\n" + "\t\t\t\t\t\t\"Type\":\t9,\n" + "\t\t\t\t\t\t\"ColId\":\t5000,\n" + "\t\t\t\t\t\t\"Bytes\":\t8\n" + "\t\t\t\t\t},\n" + "\t\t\t\t\t\"Columns\":\t[{\n" + "\t\t\t\t\t\t\t\"TableId\":\t1,\n" + "\t\t\t\t\t\t\t\"Flag\":\t0,\n" + "\t\t\t\t\t\t\t\"Info\":\t{\n" + "\t\t\t\t\t\t\t\t\"ColId\":\t1,\n" + "\t\t\t\t\t\t\t\t\"Type\":\t9,\n" + "\t\t\t\t\t\t\t\t\"Bytes\":\t8\n" + "\t\t\t\t\t\t\t}\n" + "\t\t\t\t\t\t}],\n" + "\t\t\t\t\t\"InterBytes\":\t0\n" + "\t\t\t\t},\n" + "\t\t\t\t\"Expr\":\t{\n" + "\t\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\t\"Column\":\t{\n" + "\t\t\t\t\t\t\"Type\":\t9,\n" + "\t\t\t\t\t\t\"ColId\":\t1,\n" + "\t\t\t\t\t\t\"Bytes\":\t8\n" + "\t\t\t\t\t}\n" + "\t\t\t\t}\n" + "\t\t\t}, {\n" + "\t\t\t\t\"Base\":\t{\n" + "\t\t\t\t\t\"Schema\":\t{\n" + "\t\t\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\t\t\"ColId\":\t5001,\n" + "\t\t\t\t\t\t\"Bytes\":\t4\n" + "\t\t\t\t\t},\n" + "\t\t\t\t\t\"Columns\":\t[{\n" + "\t\t\t\t\t\t\t\"TableId\":\t1,\n" + "\t\t\t\t\t\t\t\"Flag\":\t0,\n" + "\t\t\t\t\t\t\t\"Info\":\t{\n" + "\t\t\t\t\t\t\t\t\"ColId\":\t2,\n" + "\t\t\t\t\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\t\t\t\t\"Bytes\":\t4\n" + "\t\t\t\t\t\t\t}\n" + "\t\t\t\t\t\t}],\n" + "\t\t\t\t\t\"InterBytes\":\t0\n" + "\t\t\t\t},\n" + "\t\t\t\t\"Expr\":\t{\n" + "\t\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\t\"Column\":\t{\n" + "\t\t\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\t\t\"ColId\":\t2,\n" + "\t\t\t\t\t\t\"Bytes\":\t4\n" + "\t\t\t\t\t}\n" + "\t\t\t\t}\n" + "\t\t\t}],\n" + "\t\t\"InputSchema\":\t[{\n" + "\t\t\t\t\"Type\":\t9,\n" + "\t\t\t\t\"ColId\":\t5000,\n" + "\t\t\t\t\"Bytes\":\t8\n" + "\t\t\t}, {\n" + "\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\"ColId\":\t5001,\n" + "\t\t\t\t\"Bytes\":\t4\n" + "\t\t\t}],\n" + "\t\t\"TableScan\":\t{\n" + "\t\t\t\"TableId\":\t1,\n" + "\t\t\t\"TableType\":\t2,\n" + "\t\t\t\"Flag\":\t0,\n" + "\t\t\t\"Window\":\t{\n" + "\t\t\t\t\"StartKey\":\t-9.2233720368547758e+18,\n" + "\t\t\t\t\"EndKey\":\t9.2233720368547758e+18\n" + "\t\t\t}\n" + "\t\t}\n" + "\t},\n" + "\t\"DataSink\":\t{\n" + "\t\t\"Name\":\t\"Dispatch\",\n" + "\t\t\"Dispatch\":\t{\n" + "\t\t}\n" + "\t}\n" + "}"; + + SExecTaskInfo* pTaskInfo = nullptr; + int32_t code = qCreateExecTask((void*) 1, 2, NULL, (void**) &pTaskInfo); +} \ No newline at end of file diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index f440e6cdfe9678bbda242ac7b1047318e1be851a..5f17fcfaee3b401990340db062e486764909e737 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -70,6 +70,9 @@ int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) { int32_t code = qParserValidateSqlNode(&pCxt->ctx, &info, pQueryInfo, pCxt->pMsg, pCxt->msgLen); if (code == TSDB_CODE_SUCCESS) { *pQuery = (SQueryNode*)pQueryInfo; + } else { + terrno = code; + return code; } } diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index 2a50752c889619139d192595c7ffb9e3823c8790..be266bd4151249327202f2a637c6cba4cef0f19b 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -62,7 +62,7 @@ typedef struct SQueryPlanNode { SSchema *pSchema; // the schema of the input SSDatablock int32_t numOfCols; // number of input columns SArray *pExpr; // the query functions or sql aggregations - int32_t numOfExpr; // number of result columns, which is also the number of pExprs + int32_t numOfExpr; // number of result columns, which is also the number of pExprs void *pExtInfo; // additional information // children operator to generated result for current node to process // in case of join, multiple prev nodes exist. diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index a38d110d5fe50b39020a6f3ce608e5c23bf19be5..01fb3c25130b34086427976ed51f86c25290b1f0 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -88,16 +88,20 @@ static bool copySchema(SDataBlockSchema* dst, const SDataBlockSchema* src) { } static bool toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) { - dataBlockSchema->numOfCols = pPlanNode->numOfCols; - dataBlockSchema->pSchema = malloc(sizeof(SSlotSchema) * pPlanNode->numOfCols); + dataBlockSchema->numOfCols = pPlanNode->numOfExpr; + dataBlockSchema->pSchema = malloc(sizeof(SSlotSchema) * pPlanNode->numOfExpr); if (NULL == dataBlockSchema->pSchema) { return false; } - memcpy(dataBlockSchema->pSchema, pPlanNode->pSchema, sizeof(SSlotSchema) * pPlanNode->numOfCols); + dataBlockSchema->resultRowSize = 0; - for (int32_t i = 0; i < dataBlockSchema->numOfCols; ++i) { + for (int32_t i = 0; i < pPlanNode->numOfExpr; ++i) { + SExprInfo* pExprInfo = taosArrayGetP(pPlanNode->pExpr, i); + memcpy(&dataBlockSchema->pSchema[i], &pExprInfo->base.resSchema, sizeof(SSlotSchema)); + dataBlockSchema->resultRowSize += dataBlockSchema->pSchema[i].bytes; } + return true; } @@ -284,7 +288,6 @@ static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTabl return createUserTableScanNode(pPlanNode, pTable, OP_TableScan); } - static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { SQueryTableInfo* pTable = (SQueryTableInfo*)pPlanNode->pExtInfo; @@ -303,8 +306,6 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { case QNODE_TABLESCAN: node = createTableScanNode(pCxt, pPlanNode); break; - case QNODE_PROJECT: -// node = create case QNODE_MODIFY: // Insert is not an operator in a physical plan. break; diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index d8918d27def65221ff3bea28ac0eed57e1863c01..895788c55069f19eabe3ad806f690039ea94e779 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -154,9 +154,13 @@ static bool fromRawArrayWithAlloc(const cJSON* json, const char* name, FFromJson return fromItem(jArray, func, *array, itemSize, *size); } -static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void* array, int32_t itemSize, int32_t* size) { +static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void** array, int32_t itemSize, int32_t* size) { const cJSON* jArray = getArray(json, name, size); - return fromItem(jArray, func, array, itemSize, *size); + if (*array == NULL) { + *array = calloc(*size, itemSize); + } + + return fromItem(jArray, func, *array, itemSize, *size); } static char* getString(const cJSON* json, const char* name) { @@ -218,7 +222,8 @@ static bool dataBlockSchemaFromJson(const cJSON* json, void* obj) { SDataBlockSchema* schema = (SDataBlockSchema*)obj; schema->resultRowSize = getNumber(json, jkDataBlockSchemaResultRowSize); schema->precision = getNumber(json, jkDataBlockSchemaPrecision); - return fromRawArray(json, jkDataBlockSchemaSlotSchema, schemaFromJson, schema->pSchema, sizeof(SSlotSchema), &schema->numOfCols); + + return fromRawArray(json, jkDataBlockSchemaSlotSchema, schemaFromJson, (void**) &(schema->pSchema), sizeof(SSlotSchema), &schema->numOfCols); } static const char* jkColumnFilterInfoLowerRelOptr = "LowerRelOptr"; @@ -920,6 +925,8 @@ int32_t subPlanToString(const SSubplan* subplan, char** str, int32_t* len) { } *str = cJSON_Print(json); + + printf("%s\n", *str); *len = strlen(*str) + 1; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index f80a63141329e62ad08c39bcb8e0145f9f04b975..3047ef4f5ac5b8cf6eba9f3036aee44f55021d9e 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -57,34 +57,33 @@ void qDestroyQueryDag(struct SQueryDag* pDag) { } int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, uint64_t requestId) { - SQueryPlanNode* logicPlan; - int32_t code = createQueryPlan(pNode, &logicPlan); + SQueryPlanNode* pLogicPlan; + int32_t code = createQueryPlan(pNode, &pLogicPlan); if (TSDB_CODE_SUCCESS != code) { - destroyQueryPlan(logicPlan); + destroyQueryPlan(pLogicPlan); return code; } - // - if (logicPlan->info.type != QNODE_MODIFY) { -// char* str = NULL; -// queryPlanToString(logicPlan, &str); -// printf("%s\n", str); + if (pLogicPlan->info.type != QNODE_MODIFY) { + char* str = NULL; + queryPlanToString(pLogicPlan, &str); + printf("%s\n", str); } - code = optimizeQueryPlan(logicPlan); + code = optimizeQueryPlan(pLogicPlan); if (TSDB_CODE_SUCCESS != code) { - destroyQueryPlan(logicPlan); + destroyQueryPlan(pLogicPlan); return code; } - code = createDag(logicPlan, NULL, pDag, requestId); + code = createDag(pLogicPlan, NULL, pDag, requestId); if (TSDB_CODE_SUCCESS != code) { - destroyQueryPlan(logicPlan); + destroyQueryPlan(pLogicPlan); qDestroyQueryDag(*pDag); return code; } - destroyQueryPlan(logicPlan); + destroyQueryPlan(pLogicPlan); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/test/phyPlanTests.cpp b/source/libs/planner/test/phyPlanTests.cpp index 3f6f5ab77b5a5db8e69ed5d61033b48b75ddf267..2733a73a3f0dc4f6384009a91956a567ecbee35d 100644 --- a/source/libs/planner/test/phyPlanTests.cpp +++ b/source/libs/planner/test/phyPlanTests.cpp @@ -62,6 +62,9 @@ protected: } SQueryDag* dag = nullptr; uint64_t requestId = 20; + SSchema *schema = NULL; + uint32_t numOfOutput = 0; + code = qCreateQueryDag(query, &dag, requestId); dag_.reset(dag); return code; diff --git a/source/libs/qworker/CMakeLists.txt b/source/libs/qworker/CMakeLists.txt index ea3e97057b6fa340d84abe3b1070fd0acbefec27..a3db9c69929b3c1f4d60437d8c91b170cb2edcc5 100644 --- a/source/libs/qworker/CMakeLists.txt +++ b/source/libs/qworker/CMakeLists.txt @@ -1,15 +1,31 @@ aux_source_directory(src QWORKER_SRC) -add_library(qworker ${QWORKER_SRC}) +#add_library(qworker ${QWORKER_SRC}) +#target_include_directories( +# qworker +# PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/qworker" +# PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +#) +# +#target_link_libraries( +# qworker +# PRIVATE os util transport planner qcom executor +#) + +add_library(qworker STATIC ${QWORKER_SRC}) target_include_directories( qworker PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/qworker" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_link_libraries( - qworker - PRIVATE os util transport planner qcom -) +#set_target_properties(qworker PROPERTIES +# IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/libqworker.a" +# INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/libs/qworker" +# ) + +target_link_libraries(qworker + PRIVATE os util transport planner qcom executor + ) if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index 91927c370a8cb432f27d434dcae57b1a54412428..7883079fbed760c6f17b3a0d5d12013078920790 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -67,15 +67,17 @@ typedef struct SQWTaskStatus { bool drop; } SQWTaskStatus; -typedef struct SQWorkerResCache { - SRWLatch lock; - void *data; -} SQWorkerResCache; +typedef struct SQWorkerTaskHandlesCache { + SRWLatch lock; + bool needRsp; + qTaskInfo_t taskHandle; + DataSinkHandle sinkHandle; +} SQWorkerTaskHandlesCache; typedef struct SQWSchStatus { int32_t lastAccessTs; // timestamp in second SRWLatch tasksLock; - SHashObj *tasksHash; // key:queryId+taskId, value: SQWorkerTaskStatus + SHashObj *tasksHash; // key:queryId+taskId, value: SQWTaskStatus } SQWSchStatus; // Qnode/Vnode level task management @@ -83,7 +85,7 @@ typedef struct SQWorkerMgmt { SQWorkerCfg cfg; SRWLatch schLock; SRWLatch resLock; - SHashObj *schHash; //key: schedulerId, value: SQWorkerSchStatus + SHashObj *schHash; //key: schedulerId, value: SQWSchStatus SHashObj *resHash; //key: queryId+taskId, value: SQWorkerResCache } SQWorkerMgmt; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 15e894fd612e421bd131409c572d7a750723d065..2a395fcfe10962fdd48b34fceb3c6d7490650597 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -1,9 +1,12 @@ #include "qworker.h" -#include "tname.h" +#include +#include "executor.h" #include "planner.h" #include "query.h" #include "qworkerInt.h" #include "tmsg.h" +#include "tname.h" +#include "dataSinkMgt.h" int32_t qwValidateStatus(int8_t oriStatus, int8_t newStatus) { int32_t code = 0; @@ -89,15 +92,16 @@ int32_t qwUpdateTaskInfo(SQWTaskStatus *task, int8_t type, void *data) { return TSDB_CODE_SUCCESS; } -int32_t qwAddTaskResCache(SQWorkerMgmt *mgmt, uint64_t qId, uint64_t tId, void *data) { +int32_t qwAddTaskHandlesToCache(SQWorkerMgmt *mgmt, uint64_t qId, uint64_t tId, qTaskInfo_t taskHandle, DataSinkHandle sinkHandle) { char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); - SQWorkerResCache resCache = {0}; - resCache.data = data; + SQWorkerTaskHandlesCache resCache = {0}; + resCache.taskHandle = taskHandle; + resCache.sinkHandle = sinkHandle; QW_LOCK(QW_WRITE, &mgmt->resLock); - if (0 != taosHashPut(mgmt->resHash, id, sizeof(id), &resCache, sizeof(SQWorkerResCache))) { + if (0 != taosHashPut(mgmt->resHash, id, sizeof(id), &resCache, sizeof(SQWorkerTaskHandlesCache))) { QW_UNLOCK(QW_WRITE, &mgmt->resLock); qError("taosHashPut queryId[%"PRIx64"] taskId[%"PRIx64"] to resHash failed", qId, tId); return TSDB_CODE_QRY_APP_ERROR; @@ -246,13 +250,13 @@ static int32_t qwAddTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ QW_RET(code); } -static FORCE_INLINE int32_t qwAcquireTaskResCache(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t queryId, uint64_t taskId, SQWorkerResCache **res) { +static FORCE_INLINE int32_t qwAcquireTaskHandles(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t queryId, uint64_t taskId, SQWorkerTaskHandlesCache **handles) { char id[sizeof(queryId) + sizeof(taskId)] = {0}; QW_SET_QTID(id, queryId, taskId); QW_LOCK(rwType, &mgmt->resLock); - *res = taosHashGet(mgmt->resHash, id, sizeof(id)); - if (NULL == (*res)) { + *handles = taosHashGet(mgmt->resHash, id, sizeof(id)); + if (NULL == (*handles)) { QW_UNLOCK(rwType, &mgmt->resLock); return TSDB_CODE_QRY_RES_CACHE_NOT_EXIST; } @@ -602,19 +606,36 @@ int32_t qwBuildAndSendStatusRsp(SRpcMsg *pMsg, SSchedulerStatusRsp *sStatus) { return TSDB_CODE_SUCCESS; } -int32_t qwBuildAndSendFetchRsp(SRpcMsg *pMsg, void *data) { - SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); +int32_t qwInitFetchRsp(int32_t length, SRetrieveTableRsp **rsp) { + int32_t msgSize = sizeof(SRetrieveTableRsp) + length; + + SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcMallocCont(msgSize); + if (NULL == pRsp) { + qError("rpcMallocCont %d failed", msgSize); + QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + memset(pRsp, 0, sizeof(SRetrieveTableRsp)); - //TODO fill msg - pRsp->completed = true; + *rsp = pRsp; + + return TSDB_CODE_SUCCESS; +} + + +int32_t qwBuildAndSendFetchRsp(SRpcMsg *pMsg, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code) { + if (NULL == pRsp) { + pRsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); + memset(pRsp, 0, sizeof(SRetrieveTableRsp)); + dataLength = 0; + } SRpcMsg rpcRsp = { .handle = pMsg->handle, .ahandle = pMsg->ahandle, .pCont = pRsp, - .contLen = sizeof(*pRsp), - .code = 0, + .contLen = sizeof(*pRsp) + dataLength, + .code = code, }; rpcSendResponse(&rpcRsp); @@ -847,62 +868,6 @@ int32_t qwCheckTaskCancelDrop( SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryI return TSDB_CODE_SUCCESS; } -int32_t qwHandleFetch(SQWorkerResCache *res, SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg) { - SQWSchStatus *sch = NULL; - SQWTaskStatus *task = NULL; - int32_t code = 0; - int32_t needRsp = true; - void *data = NULL; - - QW_ERR_JRET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)); - QW_ERR_JRET(qwAcquireTask(QW_READ, sch, queryId, taskId, &task)); - - QW_LOCK(QW_READ, &task->lock); - - if (task->cancel || task->drop) { - qError("task is already cancelled or dropped"); - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - } - - if (task->status != JOB_TASK_STATUS_EXECUTING && task->status != JOB_TASK_STATUS_PARTIAL_SUCCEED) { - qError("invalid status %d for fetch", task->status); - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - } - - if (QW_GOT_RES_DATA(res->data)) { - data = res->data; - if (QW_LOW_RES_DATA(res->data)) { - if (task->status == JOB_TASK_STATUS_PARTIAL_SUCCEED) { - //TODO add query back to queue - } - } - } else { - if (task->status != JOB_TASK_STATUS_EXECUTING) { - qError("invalid status %d for fetch without res", task->status); - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - } - - //TODO SET FLAG FOR QUERY TO SEND RSP WHEN RES READY - - needRsp = false; - } - -_return: - if (task) { - QW_UNLOCK(QW_READ, &task->lock); - qwReleaseTask(QW_READ, sch); - } - - if (sch) { - qwReleaseScheduler(QW_READ, mgmt); - } - - if (needRsp) { - qwBuildAndSendFetchRsp(pMsg, res->data); - } - - QW_RET(code); -} int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int8_t status, int32_t errCode) { SQWSchStatus *sch = NULL; @@ -956,6 +921,113 @@ int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint6 return TSDB_CODE_SUCCESS; } + +int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg) { + SQWSchStatus *sch = NULL; + SQWTaskStatus *task = NULL; + int32_t code = 0; + int32_t needRsp = true; + void *data = NULL; + int32_t sinkStatus = 0; + int32_t dataLength = 0; + SRetrieveTableRsp *rsp = NULL; + bool queryEnd = false; + SQWorkerTaskHandlesCache *handles = NULL; + + QW_ERR_JRET(qwAcquireTaskHandles(QW_READ, mgmt, queryId, taskId, &handles)); + + QW_ERR_JRET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)); + QW_ERR_JRET(qwAcquireTask(QW_READ, sch, queryId, taskId, &task)); + + QW_LOCK(QW_READ, &task->lock); + + if (task->cancel || task->drop) { + qError("task is already cancelled or dropped"); + QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + if (task->status != JOB_TASK_STATUS_EXECUTING && task->status != JOB_TASK_STATUS_PARTIAL_SUCCEED) { + qError("invalid status %d for fetch", task->status); + QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + dsGetDataLength(handles->sinkHandle, &dataLength, &queryEnd); + + if (dataLength > 0) { + SOutputData output = {0}; + QW_ERR_JRET(qwInitFetchRsp(dataLength, &rsp)); + + output.pData = rsp->data; + + code = dsGetDataBlock(handles->sinkHandle, &output); + if (code) { + qError("dsGetDataBlock failed, code:%x", code); + QW_ERR_JRET(code); + } + + rsp->useconds = htobe64(output.useconds); + rsp->completed = 0; + rsp->precision = output.precision; + rsp->compressed = output.compressed; + rsp->compLen = htonl(dataLength); + rsp->numOfRows = htonl(output.numOfRows); + + if (DS_BUF_EMPTY == output.bufStatus && output.queryEnd) { + rsp->completed = 1; + } + + if (output.needSchedule) { + //TODO + } + + if ((!output.queryEnd) && DS_BUF_LOW == output.bufStatus) { + //TODO + //UPDATE STATUS TO EXECUTING + } + } else { + if (dataLength < 0) { + qError("invalid length from dsGetDataLength, length:%d", dataLength); + QW_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + + if (queryEnd) { + QW_ERR_JRET(qwQueryPostProcess(mgmt, sId, queryId, taskId, JOB_TASK_STATUS_SUCCEED, code)); + } else { + if (task->status != JOB_TASK_STATUS_EXECUTING) { + qError("invalid status %d for fetch without res", task->status); + QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + QW_LOCK(QW_WRITE, &handles->lock); + handles->needRsp = true; + QW_UNLOCK(QW_WRITE, &handles->lock); + + needRsp = false; + } + } + +_return: + + if (task) { + QW_UNLOCK(QW_READ, &task->lock); + qwReleaseTask(QW_READ, sch); + } + + if (sch) { + qwReleaseScheduler(QW_READ, mgmt); + } + + if (needRsp) { + qwBuildAndSendFetchRsp(pMsg, rsp, dataLength, code); + } + + if (handles) { + qwReleaseTaskResCache(QW_READ, mgmt); + } + + QW_RET(code); +} + int32_t qWorkerInit(SQWorkerCfg *cfg, void **qWorkerMgmt) { SQWorkerMgmt *mgmt = calloc(1, sizeof(SQWorkerMgmt)); if (NULL == mgmt) { @@ -1008,10 +1080,9 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { msg->taskId = htobe64(msg->taskId); msg->contentLen = ntohl(msg->contentLen); - bool queryDone = false; bool queryRsped = false; bool needStop = false; - SSubplan *plan = NULL; + struct SSubplan *plan = NULL; QW_ERR_JRET(qwCheckTaskCancelDrop(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, &needStop)); if (needStop) { @@ -1025,11 +1096,10 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_ERR_JRET(code); } - //TODO call executer to init subquery - code = 0; // return error directly - //TODO call executer to init subquery - + qTaskInfo_t pTaskInfo = NULL; + code = qCreateExecTask(node, 0, (struct SSubplan *)plan, &pTaskInfo); if (code) { + qError("qCreateExecTask failed, code:%x", code); QW_ERR_JRET(code); } else { QW_ERR_JRET(qwAddTask(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, JOB_TASK_STATUS_EXECUTING, QW_EXIST_RET_ERR, NULL, NULL)); @@ -1038,18 +1108,15 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_ERR_JRET(qwBuildAndSendQueryRsp(pMsg, TSDB_CODE_SUCCESS)); queryRsped = true; - - //TODO call executer to execute subquery - code = 0; - void *data = NULL; - queryDone = false; - //TODO call executer to execute subquery + + DataSinkHandle sinkHandle = NULL; + code = qExecTask(pTaskInfo, &sinkHandle); if (code) { + qError("qExecTask failed, code:%x", code); QW_ERR_JRET(code); } else { - QW_ERR_JRET(qwAddTaskResCache(qWorkerMgmt, msg->queryId, msg->taskId, data)); - + QW_ERR_JRET(qwAddTaskHandlesToCache(qWorkerMgmt, msg->queryId, msg->taskId, pTaskInfo, sinkHandle)); QW_ERR_JRET(qwUpdateTaskStatus(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, JOB_TASK_STATUS_PARTIAL_SUCCEED)); } @@ -1064,8 +1131,6 @@ _return: int8_t status = 0; if (TSDB_CODE_SUCCESS != code) { status = JOB_TASK_STATUS_FAILED; - } else if (queryDone) { - status = JOB_TASK_STATUS_SUCCEED; } else { status = JOB_TASK_STATUS_PARTIAL_SUCCEED; } @@ -1075,6 +1140,49 @@ _return: QW_RET(code); } +int32_t qWorkerProcessQueryContinueMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { + int32_t code = 0; + int8_t status = 0; + bool queryDone = false; + uint64_t sId, qId, tId; + + //TODO call executer to continue execute subquery + code = 0; + void *data = NULL; + queryDone = false; + //TODO call executer to continue execute subquery + + if (TSDB_CODE_SUCCESS != code) { + status = JOB_TASK_STATUS_FAILED; + } else if (queryDone) { + status = JOB_TASK_STATUS_SUCCEED; + } else { + status = JOB_TASK_STATUS_PARTIAL_SUCCEED; + } + + code = qwQueryPostProcess(qWorkerMgmt, sId, qId, tId, status, code); + + QW_RET(code); +} + + + +int32_t qWorkerProcessSinkDataMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){ + if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + + SSinkDataReq *msg = pMsg->pCont; + if (NULL == msg || pMsg->contLen < sizeof(*msg)) { + qError("invalid sink data msg"); + QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + //TODO + + return TSDB_CODE_SUCCESS; +} + int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){ if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) { return TSDB_CODE_QRY_INVALID_INPUT; @@ -1137,17 +1245,10 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_ERR_RET(qwUpdateSchLastAccess(qWorkerMgmt, msg->sId)); void *data = NULL; - SQWorkerResCache *res = NULL; int32_t code = 0; - QW_ERR_RET(qwAcquireTaskResCache(QW_READ, qWorkerMgmt, msg->queryId, msg->taskId, &res)); - - QW_ERR_JRET(qwHandleFetch(res, qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, pMsg)); - -_return: + QW_ERR_RET(qwHandleFetch(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, pMsg)); - qwReleaseTaskResCache(QW_READ, qWorkerMgmt); - QW_RET(code); } @@ -1220,31 +1321,6 @@ int32_t qWorkerProcessShowFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) QW_ERR_RET(qwBuildAndSendShowFetchRsp(pMsg, pFetchReq)); } -int32_t qWorkerContinueQuery(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { - int32_t code = 0; - int8_t status = 0; - bool queryDone = false; - uint64_t sId, qId, tId; - - //TODO call executer to continue execute subquery - code = 0; - void *data = NULL; - queryDone = false; - //TODO call executer to continue execute subquery - - if (TSDB_CODE_SUCCESS != code) { - status = JOB_TASK_STATUS_FAILED; - } else if (queryDone) { - status = JOB_TASK_STATUS_SUCCEED; - } else { - status = JOB_TASK_STATUS_PARTIAL_SUCCEED; - } - - code = qwQueryPostProcess(qWorkerMgmt, sId, qId, tId, status, code); - - QW_RET(code); -} - void qWorkerDestroy(void **qWorkerMgmt) { if (NULL == qWorkerMgmt || NULL == *qWorkerMgmt) { return; diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index d127adbcfaadcbd6fbb614f556de33e1cf4ac716..c83eba4232952fa72984251b96f4e06fde57e523 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -58,7 +58,6 @@ typedef struct SSchLevel { SArray *subTasks; // Element is SQueryTask } SSchLevel; - typedef struct SSchTask { uint64_t taskId; // task id SRWLatch lock; // task lock diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index d478e80af0070d392e9689ed29d868ff19790ad0..ed124088441ea5edc9c427d1698d87b2a3083295 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -68,7 +68,7 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXECUTING) { + if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXECUTING && SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_PARTIAL_SUCCEED) { SCH_TASK_ELOG("rsp msg conflicted with task status, status:%d, rspType:%d", SCH_GET_TASK_STATUS(pTask), msgType); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } @@ -87,68 +87,72 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { int32_t code = 0; - int8_t oriStatus = SCH_GET_JOB_STATUS(pJob); + int8_t oriStatus = 0; -/* - if (oriStatus == newStatus) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - } - - switch (oriStatus) { - case JOB_TASK_STATUS_NULL: - if (newStatus != JOB_TASK_STATUS_EXECUTING - && newStatus != JOB_TASK_STATUS_FAILED - && newStatus != JOB_TASK_STATUS_NOT_START) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - } - - break; - case JOB_TASK_STATUS_NOT_START: - if (newStatus != JOB_TASK_STATUS_CANCELLED) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - } - - break; - case JOB_TASK_STATUS_EXECUTING: - if (newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED - && newStatus != JOB_TASK_STATUS_FAILED - && newStatus != JOB_TASK_STATUS_CANCELLING - && newStatus != JOB_TASK_STATUS_CANCELLED - && newStatus != JOB_TASK_STATUS_DROPPING) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - } - - break; - case JOB_TASK_STATUS_PARTIAL_SUCCEED: - if (newStatus != JOB_TASK_STATUS_EXECUTING - && newStatus != JOB_TASK_STATUS_SUCCEED - && newStatus != JOB_TASK_STATUS_CANCELLED) { + while (true) { + oriStatus = SCH_GET_JOB_STATUS(pJob); + + if (oriStatus == newStatus) { + SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + switch (oriStatus) { + case JOB_TASK_STATUS_NULL: + if (newStatus != JOB_TASK_STATUS_NOT_START) { + SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + break; + case JOB_TASK_STATUS_NOT_START: + if (newStatus != JOB_TASK_STATUS_EXECUTING) { + SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + break; + case JOB_TASK_STATUS_EXECUTING: + if (newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED + && newStatus != JOB_TASK_STATUS_FAILED + && newStatus != JOB_TASK_STATUS_CANCELLING + && newStatus != JOB_TASK_STATUS_CANCELLED + && newStatus != JOB_TASK_STATUS_DROPPING) { + SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + break; + case JOB_TASK_STATUS_PARTIAL_SUCCEED: + if (newStatus != JOB_TASK_STATUS_FAILED + && newStatus != JOB_TASK_STATUS_SUCCEED + && newStatus != JOB_TASK_STATUS_DROPPING) { + SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + break; + case JOB_TASK_STATUS_SUCCEED: + case JOB_TASK_STATUS_FAILED: + case JOB_TASK_STATUS_CANCELLING: + if (newStatus != JOB_TASK_STATUS_DROPPING) { + SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + break; + case JOB_TASK_STATUS_CANCELLED: + case JOB_TASK_STATUS_DROPPING: SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - } - - break; - case JOB_TASK_STATUS_SUCCEED: - case JOB_TASK_STATUS_FAILED: - case JOB_TASK_STATUS_CANCELLING: - if (newStatus != JOB_TASK_STATUS_CANCELLED) { + break; + + default: + SCH_JOB_ELOG("invalid job status:%d", oriStatus); SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - } - - break; - case JOB_TASK_STATUS_CANCELLED: - case JOB_TASK_STATUS_DROPPING: - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - break; - - default: - qError("invalid task status:%d", oriStatus); - return TSDB_CODE_QRY_APP_ERROR; - } -*/ + } - SCH_SET_JOB_STATUS(pJob, newStatus); + if (oriStatus != atomic_val_compare_exchange_8(&pJob->status, oriStatus, newStatus)) { + continue; + } - SCH_JOB_DLOG("status updated from %d to %d", oriStatus, newStatus); + SCH_JOB_DLOG("job status updated from %d to %d", oriStatus, newStatus); + + break; + } return TSDB_CODE_SUCCESS; @@ -265,7 +269,6 @@ int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *ad int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { int32_t code = 0; - pJob->queryId = pDag->queryId; if (pDag->numOfSubplans <= 0) { @@ -363,7 +366,6 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { SCH_ERR_JRET(schBuildTaskRalation(pJob, planToTask)); _return: - if (planToTask) { taosHashCleanup(planToTask); } @@ -495,6 +497,33 @@ int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) { return TSDB_CODE_SUCCESS; } + +int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { + if (0 != taosHashRemove(pJob->succTasks, &pTask->taskId, sizeof(pTask->taskId))) { + SCH_TASK_WLOG("remove task from succTask list failed, may not exist, status:%d", SCH_GET_TASK_STATUS(pTask)); + } + + int32_t code = taosHashPut(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId), &pTask, POINTER_BYTES); + if (0 != code) { + if (HASH_NODE_EXIST(code)) { + *moved = true; + + SCH_TASK_ELOG("task already in execTask list, status:%d", SCH_GET_TASK_STATUS(pTask)); + SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + } + + SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:%d", errno); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + *moved = true; + + SCH_TASK_DLOG("task moved to execTask list, numOfTasks:%d", taosHashGetSize(pJob->execTasks)); + + return TSDB_CODE_SUCCESS; +} + + int32_t schTaskCheckAndSetRetry(SSchJob *job, SSchTask *task, int32_t errCode, bool *needRetry) { // TODO set retry or not based on task type/errCode/retry times/job status/available eps... // TODO if needRetry, set task retry info @@ -507,6 +536,28 @@ int32_t schTaskCheckAndSetRetry(SSchJob *job, SSchTask *task, int32_t errCode, b } + +// Note: no more error processing, handled in function internal +int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) { + // if already FAILED, no more processing + SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_FAILED)); + + if (errCode) { + atomic_store_32(&pJob->errCode, errCode); + } + + if (atomic_load_8(&pJob->userFetch) || ((!SCH_JOB_NEED_FETCH(&pJob->attr)) && pJob->attr.syncSchedule)) { + tsem_post(&pJob->rspSem); + } + + SCH_ERR_RET(atomic_load_32(&pJob->errCode)); + + assert(0); +} + + + +// Note: no more error processing, handled in function internal int32_t schFetchFromRemote(SSchJob *pJob) { int32_t code = 0; @@ -515,7 +566,13 @@ int32_t schFetchFromRemote(SSchJob *pJob) { return TSDB_CODE_SUCCESS; } - if (atomic_load_ptr(&pJob->res)) + void *res = atomic_load_ptr(&pJob->res); + if (res) { + atomic_val_compare_exchange_32(&pJob->remoteFetch, 1, 0); + + SCH_JOB_DLOG("res already fetched, res:%p", res); + return TSDB_CODE_SUCCESS; + } SCH_ERR_JRET(schBuildAndSendMsg(pJob, pJob->fetchTask, &pJob->resNode, TDMT_VND_FETCH)); @@ -525,25 +582,9 @@ _return: atomic_val_compare_exchange_32(&pJob->remoteFetch, 1, 0); - return code; -} + schProcessOnJobFailure(pJob, code); - -// Note: no more error processing, handled in function internal -int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) { - SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_FAILED)); - - if (errCode) { - atomic_store_32(&pJob->errCode, errCode); - } - - if (atomic_load_8(&pJob->userFetch) || ((!SCH_JOB_NEED_FETCH(&pJob->attr)) && pJob->attr.syncSchedule)) { - tsem_post(&pJob->rspSem); - } - - SCH_ERR_RET(atomic_load_32(&pJob->errCode)); - - assert(0); + return code; } @@ -643,7 +684,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(code); } - SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCCEED); + SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_PARTIAL_SUCCEED); SCH_ERR_JRET(schRecordTaskSucceedNode(pTask)); @@ -675,6 +716,11 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { } pJob->fetchTask = pTask; + + code = schMoveTaskToExecList(pJob, pTask, &moved); + if (code && moved) { + SCH_ERR_RET(code); + } SCH_ERR_RET(schProcessOnJobPartialSuccess(pJob)); @@ -813,7 +859,7 @@ int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, in SSchJob **job = taosHashGet(schMgmt.jobs, &pParam->queryId, sizeof(pParam->queryId)); if (NULL == job || NULL == (*job)) { - qError("QID:%"PRIx64" taosHashGet queryId not exist", pParam->queryId); + qError("QID:%"PRIx64" taosHashGet queryId not exist, may be dropped", pParam->queryId); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); } @@ -822,7 +868,10 @@ int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, in atomic_add_fetch_32(&pJob->ref, 1); int32_t s = taosHashGetSize(pJob->execTasks); - assert(s != 0); + if (s <= 0) { + qError("QID:%"PRIx64",TID:%"PRIx64" no task in execTask list", pParam->queryId, pParam->taskId); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + } SSchTask **task = taosHashGet(pJob->execTasks, &pParam->taskId, sizeof(pParam->taskId)); if (NULL == task || NULL == (*task)) { @@ -1147,7 +1196,7 @@ void schDropTaskOnExecutedNode(SSchJob *pJob, SSchTask *pTask) { int32_t size = (int32_t)taosArrayGetSize(pTask->execAddrs); if (size <= 0) { - SCH_TASK_DLOG("empty exec address, status:%d", SCH_GET_TASK_STATUS(pTask)); + SCH_TASK_DLOG("task has no exec address, no need to drop it, status:%d", SCH_GET_TASK_STATUS(pTask)); return; } @@ -1157,6 +1206,8 @@ void schDropTaskOnExecutedNode(SSchJob *pJob, SSchTask *pTask) { schBuildAndSendMsg(pJob, pTask, addr, TDMT_VND_DROP_TASK); } + + SCH_TASK_DLOG("task has %d exec address", size); } void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { @@ -1178,7 +1229,7 @@ void schDropJobAllTasks(SSchJob *pJob) { schDropTaskInHashList(pJob, pJob->failTasks); } -int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag* pDag, void** job, bool syncSchedule) { +int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** job, bool syncSchedule) { if (nodeList && taosArrayGetSize(nodeList) <= 0) { qInfo("QID:%"PRIx64" input nodeList is empty", pDag->queryId); } @@ -1291,14 +1342,14 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { return TSDB_CODE_SUCCESS; } -int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob, SQueryResult *pRes) { +int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } SSchJob *job = NULL; - SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, (void **)&job, true)); + SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, &job, true)); *pJob = job; @@ -1308,14 +1359,14 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void return TSDB_CODE_SUCCESS; } -int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob) { - if (NULL == transport || /*NULL == nodeList || */NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { +int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob) { + if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } SSchJob *job = NULL; - SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, (void **)&job, false)); + SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, &job, false)); *pJob = job; @@ -1323,14 +1374,18 @@ int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, } -int32_t scheduleFetchRows(void *job, void **data) { - if (NULL == job || NULL == data) { +int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { + if (NULL == pJob || NULL == pData) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - - SSchJob *pJob = job; int32_t code = 0; + int8_t status = SCH_GET_JOB_STATUS(pJob); + if (status == JOB_TASK_STATUS_DROPPING) { + SCH_JOB_ELOG("job is dropping, status:%d", status); + return TSDB_CODE_SCH_STATUS_ERROR; + } + atomic_add_fetch_32(&pJob->ref, 1); if (!SCH_JOB_NEED_FETCH(&pJob->attr)) { @@ -1345,14 +1400,12 @@ int32_t scheduleFetchRows(void *job, void **data) { SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - int8_t status = SCH_GET_JOB_STATUS(pJob); - if (status == JOB_TASK_STATUS_FAILED) { - *data = atomic_load_ptr(&pJob->res); + *pData = atomic_load_ptr(&pJob->res); atomic_store_ptr(&pJob->res, NULL); SCH_ERR_JRET(atomic_load_32(&pJob->errCode)); } else if (status == JOB_TASK_STATUS_SUCCEED) { - *data = atomic_load_ptr(&pJob->res); + *pData = atomic_load_ptr(&pJob->res); atomic_store_ptr(&pJob->res, NULL); goto _return; } else if (status == JOB_TASK_STATUS_PARTIAL_SUCCEED) { @@ -1365,6 +1418,7 @@ int32_t scheduleFetchRows(void *job, void **data) { if (status == JOB_TASK_STATUS_FAILED) { code = atomic_load_32(&pJob->errCode); + SCH_ERR_JRET(code); } if (pJob->res && ((SRetrieveTableRsp *)pJob->res)->completed) { @@ -1372,9 +1426,9 @@ int32_t scheduleFetchRows(void *job, void **data) { } while (true) { - *data = atomic_load_ptr(&pJob->res); + *pData = atomic_load_ptr(&pJob->res); - if (*data != atomic_val_compare_exchange_ptr(&pJob->res, *data, NULL)) { + if (*pData != atomic_val_compare_exchange_ptr(&pJob->res, *pData, NULL)) { continue; } @@ -1414,6 +1468,10 @@ void scheduleFreeJob(void *job) { return; } + schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_DROPPING); + + SCH_JOB_DLOG("job removed from list, no further ref, ref:%d", atomic_load_32(&pJob->ref)); + while (true) { int32_t ref = atomic_load_32(&pJob->ref); if (0 == ref) { @@ -1425,6 +1483,8 @@ void scheduleFreeJob(void *job) { } } + SCH_JOB_DLOG("job no ref now, status:%d", SCH_GET_JOB_STATUS(pJob)); + if (pJob->status == JOB_TASK_STATUS_EXECUTING) { schCancelJob(pJob); } @@ -1446,21 +1506,21 @@ void scheduleFreeJob(void *job) { taosArrayDestroy(pLevel->subTasks); } - taosHashCleanup(pJob->execTasks); - taosHashCleanup(pJob->failTasks); - taosHashCleanup(pJob->succTasks); - - taosArrayDestroy(pJob->levels); + taosHashCleanup(pJob->execTasks); + taosHashCleanup(pJob->failTasks); + taosHashCleanup(pJob->succTasks); - tfree(pJob->res); - - tfree(pJob); - } + taosArrayDestroy(pJob->levels); + + tfree(pJob->res); - void schedulerDestroy(void) { - if (schMgmt.jobs) { - taosHashCleanup(schMgmt.jobs); //TODO - schMgmt.jobs = NULL; - } + tfree(pJob); +} + +void schedulerDestroy(void) { + if (schMgmt.jobs) { + taosHashCleanup(schMgmt.jobs); //TODO + schMgmt.jobs = NULL; } +} diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 70e49e6b45c0b9e59ce7b6230edd182a91912136..b4385212340bf82b3acc2ee9d25a2907cfe6e6bf 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -38,52 +38,73 @@ namespace { extern "C" int32_t schHandleResponseMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode); +void schtInitLogFile() { + const char *defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; + + tsAsyncLog = 0; + qDebugFlag = 159; + + char temp[128] = {0}; + sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); + if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", tsLogDir); + } + +} + + void schtBuildQueryDag(SQueryDag *dag) { uint64_t qId = 0x0000000000000001; dag->queryId = qId; dag->numOfSubplans = 2; dag->pSubplans = taosArrayInit(dag->numOfSubplans, POINTER_BYTES); - SArray *scan = taosArrayInit(1, sizeof(SSubplan)); - SArray *merge = taosArrayInit(1, sizeof(SSubplan)); + SArray *scan = taosArrayInit(1, POINTER_BYTES); + SArray *merge = taosArrayInit(1, POINTER_BYTES); - SSubplan scanPlan = {0}; - SSubplan mergePlan = {0}; - - scanPlan.id.queryId = qId; - scanPlan.id.templateId = 0x0000000000000002; - scanPlan.id.subplanId = 0x0000000000000003; - scanPlan.type = QUERY_TYPE_SCAN; - scanPlan.execNode.numOfEps = 1; - scanPlan.execNode.nodeId = 1; - scanPlan.execNode.inUse = 0; - scanPlan.execNode.epAddr[0].port = 6030; - strcpy(scanPlan.execNode.epAddr[0].fqdn, "ep0"); - scanPlan.pChildren = NULL; - scanPlan.level = 1; - scanPlan.pParents = taosArrayInit(1, POINTER_BYTES); - scanPlan.pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode)); - - mergePlan.id.queryId = qId; - mergePlan.id.templateId = 0x4444444444; - mergePlan.id.subplanId = 0x5555555555; - mergePlan.type = QUERY_TYPE_MERGE; - mergePlan.level = 0; - mergePlan.execNode.numOfEps = 0; - mergePlan.pChildren = taosArrayInit(1, POINTER_BYTES); - mergePlan.pParents = NULL; - mergePlan.pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode)); + SSubplan *scanPlan = (SSubplan *)calloc(1, sizeof(SSubplan)); + SSubplan *mergePlan = (SSubplan *)calloc(1, sizeof(SSubplan)); + + scanPlan->id.queryId = qId; + scanPlan->id.templateId = 0x0000000000000002; + scanPlan->id.subplanId = 0x0000000000000003; + scanPlan->type = QUERY_TYPE_SCAN; + scanPlan->execNode.numOfEps = 1; + scanPlan->execNode.nodeId = 1; + scanPlan->execNode.inUse = 0; + scanPlan->execNode.epAddr[0].port = 6030; + strcpy(scanPlan->execNode.epAddr[0].fqdn, "ep0"); + scanPlan->pChildren = NULL; + scanPlan->level = 1; + scanPlan->pParents = taosArrayInit(1, POINTER_BYTES); + scanPlan->pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode)); + + mergePlan->id.queryId = qId; + mergePlan->id.templateId = 0x4444444444; + mergePlan->id.subplanId = 0x5555555555; + mergePlan->type = QUERY_TYPE_MERGE; + mergePlan->level = 0; + mergePlan->execNode.numOfEps = 0; + mergePlan->pChildren = taosArrayInit(1, POINTER_BYTES); + mergePlan->pParents = NULL; + mergePlan->pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode)); SSubplan *mergePointer = (SSubplan *)taosArrayPush(merge, &mergePlan); SSubplan *scanPointer = (SSubplan *)taosArrayPush(scan, &scanPlan); - taosArrayPush(mergePointer->pChildren, &scanPointer); - taosArrayPush(scanPointer->pParents, &mergePointer); + taosArrayPush(mergePlan->pChildren, &scanPlan); + taosArrayPush(scanPlan->pParents, &mergePlan); taosArrayPush(dag->pSubplans, &merge); taosArrayPush(dag->pSubplans, &scan); } +void schtFreeQueryDag(SQueryDag *dag) { + +} + + void schtBuildInsertDag(SQueryDag *dag) { uint64_t qId = 0x0000000000000002; @@ -138,8 +159,8 @@ int32_t schtPlanToString(const SSubplan *subplan, char** str, int32_t* len) { return 0; } -int32_t schtExecNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep) { - return 0; +void schtExecNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep) { + } @@ -196,7 +217,7 @@ void *schtSendRsp(void *param) { return NULL; } -void *pInsertJob = NULL; +struct SSchJob *pInsertJob = NULL; } @@ -207,8 +228,11 @@ TEST(queryTest, normalCase) { char *dbname = "1.db1"; char *tablename = "table1"; SVgroupInfo vgInfo = {0}; - void *pJob = NULL; + SSchJob *pJob = NULL; SQueryDag dag = {0}; + + schtInitLogFile(); + SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); SEpAddr qnodeAddr = {0}; @@ -295,6 +319,8 @@ TEST(queryTest, normalCase) { ASSERT_EQ(data, (void*)NULL); scheduleFreeJob(pJob); + + schtFreeQueryDag(&dag); } @@ -308,6 +334,9 @@ TEST(insertTest, normalCase) { SVgroupInfo vgInfo = {0}; SQueryDag dag = {0}; uint64_t numOfRows = 0; + + schtInitLogFile(); + SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); SEpAddr qnodeAddr = {0}; @@ -336,7 +365,11 @@ TEST(insertTest, normalCase) { scheduleFreeJob(pInsertJob); } +TEST(multiThread, forceFree) { + schtInitLogFile(); + +} int main(int argc, char** argv) { diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index ab3aa02f4ad17f9e226a9328cab2d9d70b3e494d..cac80c0a5f578da85b1256cc0b6d4a5ec9687c7d 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#define _DEFAULT_SOURCE #include "cJSON.h" #include "os.h" #include "taoserror.h" @@ -33,6 +32,24 @@ static inline int walBuildMetaName(SWal* pWal, int metaVer, char* buf) { return sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer); } +void* tmemmem(char* haystack, int hlen, char* needle, int nlen) { + char* limit; + + if (nlen == 0 || hlen < nlen) { + return false; + } + + limit = haystack + hlen - nlen + 1; + while ((haystack = (char*)memchr( + haystack, needle[0], limit - haystack)) != NULL) { + if (memcmp(haystack, needle, nlen) == 0) { + return haystack; + } + haystack++; + } + return NULL; +} + static inline int64_t walScanLogGetLastVer(SWal* pWal) { ASSERT(pWal->fileInfoSet != NULL); int sz = taosArrayGetSize(pWal->fileInfoSet); @@ -47,7 +64,8 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { struct stat statbuf; stat(fnameStr, &statbuf); - int readSize = MIN(WAL_MAX_SIZE, statbuf.st_size); + int readSize = MIN(WAL_MAX_SIZE + 2, statbuf.st_size); + pLastFileInfo->fileSize = statbuf.st_size; FileFd fd = taosOpenFileRead(fnameStr); if (fd < 0) { @@ -64,6 +82,7 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { return -1; } + taosLSeekFile(fd, -readSize, SEEK_END); if (readSize != taosReadFile(fd, buf, readSize)) { free(buf); taosCloseFile(fd); @@ -71,21 +90,25 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { return -1; } - char* found = strstr(buf, (const char*)&magic); - if (found == NULL) { - ASSERT(false); - // file has to be deleted - free(buf); - taosCloseFile(fd); - terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - return -1; - } - char *another; - while((another = strstr(found + 1, (const char*)&magic)) != NULL) { + char* haystack = buf; + char* found = NULL; + char *candidate; + while((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(uint64_t))) != NULL) { // read and validate - SWalHead *logContent = (SWalHead*)another; + SWalHead *logContent = (SWalHead*)candidate; if (walValidHeadCksum(logContent) == 0 && walValidBodyCksum(logContent) == 0) { - found = another; + found = candidate; + } + haystack = candidate + 1; + } + if (found == buf) { + SWalHead *logContent = (SWalHead*)found; + if (walValidHeadCksum(logContent) != 0 || walValidBodyCksum(logContent) != 0) { + // file has to be deleted + free(buf); + taosCloseFile(fd); + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + return -1; } } taosCloseFile(fd); @@ -120,7 +143,6 @@ int walCheckAndRepairMeta(SWal* pWal) { SWalFileInfo fileInfo; memset(&fileInfo, -1, sizeof(SWalFileInfo)); sscanf(name, "%" PRId64 ".log", &fileInfo.firstVer); - FileFd fd = taosOpenFileRead(ent->d_name); //get lastVer //get size taosArrayPush(pLogInfoArray, &fileInfo); @@ -137,28 +159,25 @@ int walCheckAndRepairMeta(SWal* pWal) { } int newSz = taosArrayGetSize(pLogInfoArray); // case 1. meta file not exist / cannot be parsed - if (pWal->fileInfoSet == NULL && newSz != 0) { - // recover fileInfo set - pWal->fileInfoSet = pLogInfoArray; - if (newSz != 0) { - // recover meta version - pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pLogInfoArray, 0))->firstVer; - pWal->writeCur = newSz - 1; - } - // recover file size - } else if (oldSz < newSz) { + if (oldSz < newSz) { for (int i = oldSz; i < newSz; i++) { SWalFileInfo *pFileInfo = taosArrayGet(pLogInfoArray, i); taosArrayPush(pWal->fileInfoSet, pFileInfo); } + pWal->writeCur = newSz - 1; - } - - if (pWal->fileInfoSet && taosArrayGetSize(pWal->fileInfoSet) != 0) { + pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pLogInfoArray, 0))->firstVer; pWal->vers.lastVer = walScanLogGetLastVer(pWal); + ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer = pWal->vers.lastVer; ASSERT(pWal->vers.lastVer != -1); - } + int code = walSaveMeta(pWal); + if (code < 0) { + taosArrayDestroy(pLogInfoArray); + return -1; + } + } + // case 2. versions in meta not match log // or some log not included in meta // (e.g. program killed) @@ -182,14 +201,11 @@ int walCheckAndRepairMeta(SWal* pWal) { } #endif - int code = walSaveMeta(pWal); - if (code < 0) { - return -1; - } // get last version of this file // // rebuild meta + taosArrayDestroy(pLogInfoArray); return 0; } @@ -397,6 +413,10 @@ int walLoadMeta(SWal* pWal) { } memset(buf, 0, size + 5); FileFd fd = taosOpenFileRead(fnameStr); + if (fd < 0) { + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + return -1; + } if (taosReadFile(fd, buf, size) != size) { terrno = TAOS_SYSTEM_ERROR(errno); taosCloseFile(fd); diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index d12acb52c6aa9f9f56f89b5244d04cc8341067f9..d5c28d9d9bb6bf76300efeb7ced7d2d38c01272b 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -106,6 +106,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { // init write buffer memset(&pWal->writeHead, 0, sizeof(SWalHead)); pWal->writeHead.head.headVer = WAL_HEAD_VER; + pWal->writeHead.magic = WAL_MAGIC; if (pthread_mutex_init(&pWal->mutex, NULL) < 0) { taosArrayDestroy(pWal->fileInfoSet); @@ -121,7 +122,9 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { return NULL; } - if (walLoadMeta(pWal) < 0 && walCheckAndRepairMeta(pWal) < 0) { + walLoadMeta(pWal); + + if (walCheckAndRepairMeta(pWal) < 0) { taosRemoveRef(tsWal.refSetId, pWal->refId); pthread_mutex_destroy(&pWal->mutex); taosArrayDestroy(pWal->fileInfoSet); @@ -130,6 +133,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { } if (walCheckAndRepairIdx(pWal) < 0) { + } wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level, diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index a95c75b11d5bfe971b1cee5d78d2a1ea54e2c46a..b65a200ca15f18fe4175d7dc109597a7ca3f0db6 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -340,7 +340,10 @@ TEST_F(WalRetentionEnv, repairMeta1) { char buf[100]; sprintf(buf, "%s/meta-ver%d", pathName, 0); remove(buf); + sprintf(buf, "%s/meta-ver%d", pathName, 1); + remove(buf); SetUp(); + //getchar(); ASSERT_EQ(pWal->vers.lastVer, 99); @@ -377,4 +380,26 @@ TEST_F(WalRetentionEnv, repairMeta1) { ASSERT_EQ(code, 0); } + for (int i = 0; i < 1000; i++) { + int ver = rand() % 200; + code = walReadWithHandle(pRead, ver); + ASSERT_EQ(code, 0); + + // printf("rrbody: \n"); + // for(int i = 0; i < pRead->pHead->head.len; i++) { + // printf("%d ", pRead->pHead->head.body[i]); + //} + // printf("\n"); + + ASSERT_EQ(pRead->pHead->head.version, ver); + ASSERT_EQ(pRead->curVersion, ver + 1); + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, ver); + int len = strlen(newStr); + ASSERT_EQ(pRead->pHead->head.len, len); + for (int j = 0; j < len; j++) { + EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); + } + } + } diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index e235b0714e181814e66b709f833ffaf7c5c98d9c..0be17ca2b9299a4477d58f0527e0c8658e07c987 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -1087,7 +1087,7 @@ int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { return 0; } - return -1; + return 0; } char *taosGetCmdlineByPID(int pid) { diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 65c8a45d009c2dfb7cfe592a226a405f3e709e26..130628e7996f35c92137d6d735dca64933c170ff 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -274,7 +274,7 @@ TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STable TsdbQueryHandleT tsdbQueryCacheLast(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pMemRef); -bool isTsdbCacheLastRow(TsdbQueryHandleT* pQueryHandle); +bool isTsdbCacheLastRow(TsdbQueryHandleT* pTsdbReadHandle); /** @@ -308,19 +308,19 @@ int64_t tsdbGetNumOfRowsInMemTable(TsdbQueryHandleT* pHandle); /** * move to next block if exists * - * @param pQueryHandle + * @param pTsdbReadHandle * @return */ -bool tsdbNextDataBlock(TsdbQueryHandleT pQueryHandle); +bool tsdbNextDataBlock(TsdbQueryHandleT pTsdbReadHandle); /** * Get current data block information * - * @param pQueryHandle + * @param pTsdbReadHandle * @param pBlockInfo * @return */ -void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle, SDataBlockInfo *pBlockInfo); +void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo); /** * @@ -332,7 +332,7 @@ void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle, SDataBlockInfo *p * @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0 * @return */ -int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT *pQueryHandle, SDataStatis **pBlockStatis); +int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT *pTsdbReadHandle, SDataStatis **pBlockStatis); /** * @@ -340,11 +340,11 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT *pQueryHandle, SDataSta * the returned data block must be satisfied with the time window condition in any cases, * which means the SData data block is not actually the completed disk data blocks. * - * @param pQueryHandle query handle + * @param pTsdbReadHandle query handle * @param pColumnIdList required data columns id list * @return */ -SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pQueryHandle, SArray *pColumnIdList); +SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pTsdbReadHandle, SArray *pColumnIdList); /** * Get the qualified table id for a super table according to the tag query expression. diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 19ca8e7ed8867f32c9625fe66ad4c04d64bc98e7..0c0e3363c89d034225098b1dee36e3cb67ea31c1 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -284,7 +284,7 @@ typedef struct SQueryRuntimeEnv { uint32_t status; // query status void* qinfo; uint8_t scanFlag; // denotes reversed scan of data or not - void* pQueryHandle; + void* pTsdbReadHandle; int32_t prevGroupId; // previous executed group id bool enableGroupData; @@ -418,7 +418,7 @@ typedef struct SQueryParam { } SQueryParam; typedef struct STableScanInfo { - void *pQueryHandle; + void *pTsdbReadHandle; int32_t numOfBlocks; int32_t numOfSkipped; int32_t numOfBlockStatis; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index be1bfb81437c345361941b890991c33779f907de..a6a9115b2f0c073e008fc316769279a33a155d49 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -2326,8 +2326,8 @@ _clean: static void doFreeQueryHandle(SQueryRuntimeEnv* pRuntimeEnv) { SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); - pRuntimeEnv->pQueryHandle = NULL; + tsdbCleanupQueryHandle(pRuntimeEnv->pTsdbReadHandle); + pRuntimeEnv->pTsdbReadHandle = NULL; SMemRef* pMemRef = &pQueryAttr->memRef; assert(pMemRef->ref == 0 && pMemRef->snapshot.imem == NULL && pMemRef->snapshot.mem == NULL); @@ -3148,10 +3148,10 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa } else if ((*status) == BLK_DATA_STATIS_NEEDED) { // this function never returns error? pCost->loadBlockStatis += 1; - tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockStatis); + tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockStatis); if (pBlock->pBlockStatis == NULL) { // data block statistics does not exist, load data block - pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); + pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); pCost->totalCheckedRows += pBlock->info.rows; } } else { @@ -3159,7 +3159,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa // load the data block statistics to perform further filter pCost->loadBlockStatis += 1; - tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockStatis); + tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockStatis); if (pQueryAttr->topBotQuery && pBlock->pBlockStatis != NULL) { { // set previous window @@ -3205,7 +3205,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa pCost->totalCheckedRows += pBlockInfo->rows; pCost->loadBlocks += 1; - pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); + pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); if (pBlock->pDataBlock == NULL) { return terrno; } @@ -4494,7 +4494,7 @@ void queryCostStatis(SQInfo *pQInfo) { // // assert(pQueryAttr->pos >= 0 && pQueryAttr->pos <= pBlockInfo->rows - 1); // -// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // // // update the pQueryAttr->limit.offset value, and pQueryAttr->pos value @@ -4521,15 +4521,15 @@ void queryCostStatis(SQInfo *pQInfo) { // int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); // // STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; -// TsdbQueryHandleT pQueryHandle = pRuntimeEnv->pQueryHandle; +// TsdbQueryHandleT pTsdbReadHandle = pRuntimeEnv->pTsdbReadHandle; // // SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; -// while (tsdbNextDataBlock(pQueryHandle)) { +// while (tsdbNextDataBlock(pTsdbReadHandle)) { // if (isQueryKilled(pRuntimeEnv->qinfo)) { // longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); // } // -// tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); +// tsdbRetrieveDataBlockInfo(pTsdbReadHandle, &blockInfo); // // if (pQueryAttr->limit.offset > blockInfo.rows) { // pQueryAttr->limit.offset -= blockInfo.rows; @@ -4562,7 +4562,7 @@ void queryCostStatis(SQInfo *pQInfo) { // // // load the data block and check data remaining in current data block // // TODO optimize performance -// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // // tw = *win; @@ -4627,8 +4627,8 @@ void queryCostStatis(SQInfo *pQInfo) { // STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; // // SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; -// while (tsdbNextDataBlock(pRuntimeEnv->pQueryHandle)) { -// tsdbRetrieveDataBlockInfo(pRuntimeEnv->pQueryHandle, &blockInfo); +// while (tsdbNextDataBlock(pRuntimeEnv->pTsdbReadHandle)) { +// tsdbRetrieveDataBlockInfo(pRuntimeEnv->pTsdbReadHandle, &blockInfo); // // if (QUERY_IS_ASC_QUERY(pQueryAttr)) { // if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { @@ -4674,7 +4674,7 @@ void queryCostStatis(SQInfo *pQInfo) { // */ // if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) { // -// SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // // if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) { @@ -4748,7 +4748,7 @@ static int32_t setupQueryHandle(void* tsdb, SQueryRuntimeEnv* pRuntimeEnv, int64 terrno = TSDB_CODE_SUCCESS; if (isFirstLastRowQuery(pQueryAttr)) { - pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryLastRow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); // update the query time window pQueryAttr->window = cond.twindow; @@ -4769,11 +4769,11 @@ static int32_t setupQueryHandle(void* tsdb, SQueryRuntimeEnv* pRuntimeEnv, int64 } } } else if (isCachedLastQuery(pQueryAttr)) { - pRuntimeEnv->pQueryHandle = tsdbQueryCacheLast(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryCacheLast(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); } else if (pQueryAttr->pointInterpQuery) { - pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); } else { - pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryTables(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); } return terrno; @@ -4831,19 +4831,19 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr switch(tbScanner) { case OP_TableBlockInfoScan: { - pRuntimeEnv->proot = createTableBlockInfoScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); + pRuntimeEnv->proot = createTableBlockInfoScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv); break; } case OP_TableSeqScan: { - pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); + pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv); break; } case OP_DataBlocksOptScan: { - pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0); + pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0); break; } case OP_TableScan: { - pRuntimeEnv->proot = createTableScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr)); + pRuntimeEnv->proot = createTableScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr)); break; } default: { // do nothing @@ -4974,13 +4974,13 @@ static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { *newgroup = false; - while (tsdbNextDataBlock(pTableScanInfo->pQueryHandle)) { + while (tsdbNextDataBlock(pTableScanInfo->pTsdbReadHandle)) { if (isQueryKilled(pOperator->pRuntimeEnv->qinfo)) { longjmp(pOperator->pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } pTableScanInfo->numOfBlocks += 1; - tsdbRetrieveDataBlockInfo(pTableScanInfo->pQueryHandle, &pBlock->info); + tsdbRetrieveDataBlockInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->info); // todo opt if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) { @@ -5037,7 +5037,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { } if (++pTableScanInfo->current >= pTableScanInfo->times) { - if (pTableScanInfo->reverseTimes <= 0 || isTsdbCacheLastRow(pTableScanInfo->pQueryHandle)) { + if (pTableScanInfo->reverseTimes <= 0 || isTsdbCacheLastRow(pTableScanInfo->pTsdbReadHandle)) { return NULL; } else { break; @@ -5046,7 +5046,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { // do prepare for the next round table scan operation STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); - tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); + tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond); setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); pRuntimeEnv->scanFlag = REPEAT_SCAN; @@ -5069,7 +5069,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { setupEnvForReverseScan(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); - tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); + tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond); qDebug("QInfo:0x%"PRIx64" start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey); @@ -5112,8 +5112,8 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { tableBlockDist.maxRows = INT_MIN; tableBlockDist.minRows = INT_MAX; - tsdbGetFileBlocksDistInfo(pTableScanInfo->pQueryHandle, &tableBlockDist); - tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pQueryHandle); + tsdbGetFileBlocksDistInfo(pTableScanInfo->pTsdbReadHandle, &tableBlockDist); + tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pTsdbReadHandle); SSDataBlock* pBlock = &pTableScanInfo->block; pBlock->info.rows = 1; @@ -5142,7 +5142,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* assert(repeatTime > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->times = repeatTime; pInfo->reverseTimes = 0; pInfo->order = pRuntimeEnv->pQueryAttr->order.order; @@ -5165,7 +5165,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) { STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->times = 1; pInfo->reverseTimes = 0; pInfo->order = pRuntimeEnv->pQueryAttr->order.order; @@ -5189,7 +5189,7 @@ SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeE SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) { STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); SColumnInfoData infoData = {{0}}; @@ -5271,7 +5271,7 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntime assert(repeatTime > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->times = repeatTime; pInfo->reverseTimes = reverseTime; pInfo->current = 0; diff --git a/tests/script/sim/db/basic1.sim b/tests/script/sim/db/basic1.sim index 52af7d93ea126e5d54b059b400ac03a09a4211ae..33af1c5b597d0b165357397dd3326a1285b711e9 100644 --- a/tests/script/sim/db/basic1.sim +++ b/tests/script/sim/db/basic1.sim @@ -85,7 +85,6 @@ if $data02 != 2 then return -1 endi -return system sh/exec.sh -n dnode1 -s stop -x SIGKILL system sh/exec.sh -n dnode1 -s start @@ -104,4 +103,4 @@ if $rows != 2 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index f2db9d0a0cf6781f7b4b8205f90a0dc3a9e8f7e1..a5a55bbc9162c5dcd8e1843ebc55cd2a89a4b7c7 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -40,6 +40,8 @@ typedef struct { float createTableSpeed; float insertDataSpeed; int64_t startMs; + int64_t maxDelay; + int64_t minDelay; pthread_t thread; } SThreadInfo; @@ -58,12 +60,30 @@ int32_t main(int32_t argc, char *argv[]) { pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); SThreadInfo *pInfo = (SThreadInfo *)calloc(numOfThreads, sizeof(SThreadInfo)); - int64_t numOfTablesPerThread = numOfTables / numOfThreads; - numOfTables = numOfTablesPerThread * numOfThreads; + //int64_t numOfTablesPerThread = numOfTables / numOfThreads; + //numOfTables = numOfTablesPerThread * numOfThreads; + + + if (numOfThreads < 1) { + numOfThreads = 1; + } + + int64_t a = numOfTables / numOfThreads; + if (a < 1) { + numOfThreads = numOfTables; + a = 1; + } + + int64_t b = 0; + b = numOfTables % numOfThreads; + + int64_t tableFrom = 0; for (int32_t i = 0; i < numOfThreads; ++i) { - pInfo[i].tableBeginIndex = i * numOfTablesPerThread; - pInfo[i].tableEndIndex = (i + 1) * numOfTablesPerThread; + pInfo[i].tableBeginIndex = tableFrom; + pInfo[i].tableEndIndex = i < b ? tableFrom + a : tableFrom + a - 1; + tableFrom = pInfo[i].tableEndIndex + 1; pInfo[i].threadIndex = i; + pInfo[i].minDelay = INT64_MAX; strcpy(pInfo[i].dbName, dbName); strcpy(pInfo[i].stbName, stbName); pthread_create(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i)); @@ -74,9 +94,15 @@ int32_t main(int32_t argc, char *argv[]) { pthread_join(pInfo[i].thread, NULL); } + int64_t maxDelay = 0; + int64_t minDelay = INT64_MAX; + float createTableSpeed = 0; for (int32_t i = 0; i < numOfThreads; ++i) { createTableSpeed += pInfo[i].createTableSpeed; + + if (pInfo[i].maxDelay > maxDelay) maxDelay = pInfo[i].maxDelay; + if (pInfo[i].minDelay < minDelay) minDelay = pInfo[i].minDelay; } float insertDataSpeed = 0; @@ -84,10 +110,19 @@ int32_t main(int32_t argc, char *argv[]) { insertDataSpeed += pInfo[i].insertDataSpeed; } - pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d %s", GREEN, numOfTables, createTableSpeed, - numOfThreads, NC); - pPrint("%s total %" PRId64 " tables, %.1f rows/second, threads:%d %s", GREEN, numOfTables, insertDataSpeed, + pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d, maxDelay: %" PRId64 "us, minDelay: %" PRId64 "us %s", + GREEN, + numOfTables, + createTableSpeed, + numOfThreads, + maxDelay, + minDelay, + NC); + + if (insertData) { + pPrint("%s total %" PRId64 " tables, %.1f rows/second, threads:%d %s", GREEN, numOfTables, insertDataSpeed, numOfThreads, NC); + } pthread_attr_destroy(&thattr); free(pInfo); @@ -99,36 +134,36 @@ void createDbAndStb() { TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); if (con == NULL) { - pError("failed to connect to DB, reason:%s", taos_errstr(con)); + pError("failed to connect to DB, reason:%s", taos_errstr(NULL)); exit(1); } sprintf(qstr, "create database if not exists %s vgroups %d", dbName, numOfVgroups); - TAOS_RES *pSql = taos_query(con, qstr); - int32_t code = taos_errno(pSql); + TAOS_RES *pRes = taos_query(con, qstr); + int32_t code = taos_errno(pRes); if (code != 0) { - pError("failed to create database:%s, sql:%s, code:%d reason:%s", dbName, qstr, taos_errno(con), taos_errstr(con)); + pError("failed to create database:%s, sql:%s, code:%d reason:%s", dbName, qstr, taos_errno(pRes), taos_errstr(pRes)); exit(0); } - taos_free_result(pSql); + taos_free_result(pRes); sprintf(qstr, "use %s", dbName); - pSql = taos_query(con, qstr); - code = taos_errno(pSql); + pRes = taos_query(con, qstr); + code = taos_errno(pRes); if (code != 0) { - pError("failed to use db, code:%d reason:%s", taos_errno(con), taos_errstr(con)); + pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); exit(0); } - taos_free_result(pSql); + taos_free_result(pRes); sprintf(qstr, "create table %s (ts timestamp, i int) tags (j int)", stbName); - pSql = taos_query(con, qstr); - code = taos_errno(pSql); + pRes = taos_query(con, qstr); + code = taos_errno(pRes); if (code != 0) { - pError("failed to use db, code:%d reason:%s", taos_errno(con), taos_errstr(con)); + pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); exit(0); } - taos_free_result(pSql); + taos_free_result(pRes); taos_close(con); } @@ -160,16 +195,20 @@ void *threadFunc(void *param) { TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); if (con == NULL) { - pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con)); + pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(NULL)); exit(1); } + //printf("thread:%d, table range: %"PRId64 " - %"PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, pInfo->tableEndIndex); sprintf(qstr, "use %s", pInfo->dbName); - TAOS_RES *pSql = taos_query(con, qstr); - taos_free_result(pSql); + TAOS_RES *pRes = taos_query(con, qstr); + taos_free_result(pRes); + if (createTable) { - pInfo->startMs = taosGetTimestampMs(); + int64_t curMs = 0; + int64_t beginMs = taosGetTimestampMs(); + pInfo->startMs = beginMs; for (int64_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) { int64_t batch = (pInfo->tableEndIndex - t); batch = MIN(batch, batchNum); @@ -179,14 +218,22 @@ void *threadFunc(void *param) { len += sprintf(qstr + len, " t%" PRId64 " using %s tags(%" PRId64 ")", t + i, stbName, t + i); } - TAOS_RES *pSql = taos_query(con, qstr); - code = taos_errno(pSql); + int64_t startTs = taosGetTimestampUs(); + TAOS_RES *pRes = taos_query(con, qstr); + code = taos_errno(pRes); if (code != 0) { pError("failed to create table t%" PRId64 ", reason:%s", t, tstrerror(code)); } - taos_free_result(pSql); - - if (t % 100000 == 0) { + taos_free_result(pRes); + int64_t endTs = taosGetTimestampUs(); + int64_t delay = endTs - startTs; + //printf("==== %"PRId64" - %"PRId64", %"PRId64"\n", startTs, endTs, delay); + if (delay > pInfo->maxDelay) pInfo->maxDelay = delay; + if (delay < pInfo->minDelay) pInfo->minDelay = delay; + + curMs = taosGetTimestampMs(); + if (curMs - beginMs > 10000) { + beginMs = curMs; printCreateProgress(pInfo, t); } t += (batch - 1); @@ -195,6 +242,9 @@ void *threadFunc(void *param) { } if (insertData) { + int64_t curMs = 0; + int64_t beginMs = taosGetTimestampMs();; + pInfo->startMs = taosGetTimestampMs(); for (int64_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) { int64_t batch = (pInfo->tableEndIndex - t); @@ -205,14 +255,15 @@ void *threadFunc(void *param) { len += sprintf(qstr + len, " t%" PRId64 " values(now, %" PRId64 ")", t + i, t + i); } - TAOS_RES *pSql = taos_query(con, qstr); - code = taos_errno(pSql); + TAOS_RES *pRes = taos_query(con, qstr); + code = taos_errno(pRes); if (code != 0) { pError("failed to insert table t%" PRId64 ", reason:%s", t, tstrerror(code)); } - taos_free_result(pSql); + taos_free_result(pRes); - if (t % 100000 == 0) { + curMs = taosGetTimestampMs(); + if (curMs - beginMs > 10000) { printInsertProgress(pInfo, t); } t += (batch - 1); @@ -266,7 +317,7 @@ void parseArgument(int32_t argc, char *argv[]) { numOfThreads = atoi(argv[++i]); } else if (strcmp(argv[i], "-n") == 0) { numOfTables = atoll(argv[++i]); - } else if (strcmp(argv[i], "-n") == 0) { + } else if (strcmp(argv[i], "-v") == 0) { numOfVgroups = atoi(argv[++i]); } else if (strcmp(argv[i], "-a") == 0) { createTable = atoi(argv[++i]);