Skip to content

T
TDengine

taosdata / TDengine
大约 2 年 前同步成功

通知 1192
Star 22018
Fork 4786
T
TDengine
未验证 提交 369b5559 编写于 作者: S Shengliang Guan 提交者： GitHub
浏览文件
操作

Merge pull request #9744 from taosdata/feature/qnode 

Feature/qnode
上级 442f0db1 6677dfba
隐藏空白更改
内联 并排
Showing with 2824 addition and 2321 deletion
include/common/common.h
浏览文件 @ 369b5559
...@@ -62,6 +62,12 @@ typedef struct SConstantItem { ...@@ -62,6 +62,12 @@ typedef struct SConstantItem {
SVariant value; SVariant value;
} SConstantItem; } SConstantItem;
typedef struct {
uint32_t numOfTables;
SArray *pGroupList;
SHashObj *map; // speedup acquire the tableQueryInfo by table uid
} STableGroupInfo;
typedef struct SSDataBlock { typedef struct SSDataBlock {
SColumnDataAgg *pBlockAgg; SColumnDataAgg *pBlockAgg;
SArray *pDataBlock; // SArray<SColumnInfoData> SArray *pDataBlock; // SArray<SColumnInfoData>
......
include/common/tmsg.h
浏览文件 @ 369b5559
...@@ -986,6 +986,14 @@ typedef struct { ...@@ -986,6 +986,14 @@ typedef struct {
char msg[]; char msg[];
} SSubQueryMsg; } SSubQueryMsg;
typedef struct {
SMsgHead header;
uint64_t sId;
uint64_t queryId;
uint64_t taskId;
} SSinkDataReq;
typedef struct { typedef struct {
SMsgHead header; SMsgHead header;
uint64_t sId; uint64_t sId;
......
include/dnode/vnode/tsdb/tsdb.h
浏览文件 @ 369b5559
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
#include "mallocator.h" #include "mallocator.h"
#include "meta.h" #include "meta.h"
#include "common.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
...@@ -39,6 +40,10 @@ typedef struct STable { ...@@ -39,6 +40,10 @@ typedef struct STable {
STSchema *pSchema; STSchema *pSchema;
} STable; } STable;
#define BLOCK_LOAD_OFFSET_SEQ_ORDER 1
#define BLOCK_LOAD_TABLE_SEQ_ORDER 2
#define BLOCK_LOAD_TABLE_RR_ORDER 3
#define TABLE_TID(t) (t)->tid #define TABLE_TID(t) (t)->tid
#define TABLE_UID(t) (t)->uid #define TABLE_UID(t) (t)->uid
...@@ -58,6 +63,22 @@ typedef struct STsdbCfg { ...@@ -58,6 +63,22 @@ typedef struct STsdbCfg {
int8_t compression; int8_t compression;
} STsdbCfg; } STsdbCfg;
// query condition to build multi-table data block iterator
typedef struct STsdbQueryCond {
STimeWindow twindow;
int32_t order; // desc|asc order to iterate the data block
int32_t numOfCols;
SColumnInfo *colList;
bool loadExternalRows; // load external rows or not
int32_t type; // data block load type:
} STsdbQueryCond;
typedef struct {
void *pTable;
TSKEY lastKey;
uint64_t uid;
} STableKeyInfo;
// STsdb // STsdb
STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta); STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta);
void tsdbClose(STsdb *); void tsdbClose(STsdb *);
...@@ -70,6 +91,119 @@ int tsdbCommit(STsdb *pTsdb); ...@@ -70,6 +91,119 @@ int tsdbCommit(STsdb *pTsdb);
int tsdbOptionsInit(STsdbCfg *); int tsdbOptionsInit(STsdbCfg *);
void tsdbOptionsClear(STsdbCfg *); void tsdbOptionsClear(STsdbCfg *);
typedef void* tsdbReadHandleT;
/**
* Get the data block iterator, starting from position according to the query condition
*
* @param tsdb tsdb handle
* @param pCond query condition, including time window, result set order, and basic required columns for each block
* @param tableInfoGroup table object list in the form of set, grouped into different sets according to the
* group by condition
* @param qinfo query info handle from query processor
* @return
*/
tsdbReadHandleT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId,
void *pRef);
/**
* Get the last row of the given query time window for all the tables in STableGroupInfo object.
* Note that only one data block with only row will be returned while invoking retrieve data block function for
* all tables in this group.
*
* @param tsdb tsdb handle
* @param pCond query condition, including time window, result set order, and basic required columns for each block
* @param tableInfo table list.
* @return
*/
//tsdbReadHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, uint64_t qId,
// SMemRef *pRef);
tsdbReadHandleT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, void* pMemRef);
bool isTsdbCacheLastRow(tsdbReadHandleT* pTsdbReadHandle);
/**
* get num of rows in mem table
*
* @param pHandle
* @return row size
*/
int64_t tsdbGetNumOfRowsInMemTable(tsdbReadHandleT* pHandle);
/**
* move to next block if exists
*
* @param pTsdbReadHandle
* @return
*/
bool tsdbNextDataBlock(tsdbReadHandleT pTsdbReadHandle);
/**
* Get current data block information
*
* @param pTsdbReadHandle
* @param pBlockInfo
* @return
*/
void tsdbRetrieveDataBlockInfo(tsdbReadHandleT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo);
/**
*
* Get the pre-calculated information w.r.t. current data block.
*
* In case of data block in cache, the pBlockStatis will always be NULL.
* If a block is not completed loaded from disk, the pBlockStatis will be NULL.
* @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0
* @return
*/
int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReadHandleT *pTsdbReadHandle, SDataStatis **pBlockStatis);
/**
*
* The query condition with primary timestamp is passed to iterator during its constructor function,
* the returned data block must be satisfied with the time window condition in any cases,
* which means the SData data block is not actually the completed disk data blocks.
*
* @param pTsdbReadHandle query handle
* @param pColumnIdList required data columns id list
* @return
*/
SArray *tsdbRetrieveDataBlock(tsdbReadHandleT *pTsdbReadHandle, SArray *pColumnIdList);
/**
* destroy the created table group list, which is generated by tag query
* @param pGroupList
*/
void tsdbDestroyTableGroup(STableGroupInfo *pGroupList);
/**
* create the table group result including only one table, used to handle the normal table query
*
* @param tsdb tsdbHandle
* @param uid table uid
* @param pGroupInfo the generated result
* @return
*/
int32_t tsdbGetOneTableGroup(STsdb *tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo);
/**
*
* @param tsdb
* @param pTableIdList
* @param pGroupInfo
* @return
*/
int32_t tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo);
/**
* clean up the query handle
* @param queryHandle
*/
void tsdbCleanupQueryHandle(tsdbReadHandleT queryHandle);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
......
include/libs/executor/dataSinkMgt.h
浏览文件 @ 369b5559
...@@ -22,6 +22,7 @@ extern "C" { ...@@ -22,6 +22,7 @@ extern "C" {
#include "os.h" #include "os.h"
#include "thash.h" #include "thash.h"
#include "executor.h"
#define DS_BUF_LOW 1 #define DS_BUF_LOW 1
#define DS_BUF_FULL 2 #define DS_BUF_FULL 2
...@@ -31,14 +32,12 @@ struct SDataSink; ...@@ -31,14 +32,12 @@ struct SDataSink;
struct SSDataBlock; struct SSDataBlock;
typedef struct SDataSinkMgtCfg { typedef struct SDataSinkMgtCfg {
uint32_t maxDataBlockNum; uint32_t maxDataBlockNum; // todo: this should be numOfRows?
uint32_t maxDataBlockNumPerQuery; uint32_t maxDataBlockNumPerQuery;
} SDataSinkMgtCfg; } SDataSinkMgtCfg;
int32_t dsDataSinkMgtInit(SDataSinkMgtCfg *cfg); int32_t dsDataSinkMgtInit(SDataSinkMgtCfg *cfg);
typedef void* DataSinkHandle;
typedef struct SInputData { typedef struct SInputData {
const struct SSDataBlock* pData; const struct SSDataBlock* pData;
SHashObj* pTableRetrieveTsMap; SHashObj* pTableRetrieveTsMap;
......
include/libs/executor/executor.h
浏览文件 @ 369b5559
...@@ -21,24 +21,30 @@ extern "C" { ...@@ -21,24 +21,30 @@ extern "C" {
#endif #endif
typedef void* qTaskInfo_t; typedef void* qTaskInfo_t;
typedef void* DataSinkHandle;
struct SSubplan;
/**
* create the qinfo object according to QueryTableMsg /**
* @param tsdb * Create the exec task object according to task json
* @param pQueryTableMsg * @param tsdb
* @param pTaskInfo * @param vgId
* @return * @param pTaskInfoMsg
*/ * @param pTaskInfo
int32_t qCreateTask(void* tsdb, int32_t vgId, void* pQueryTableMsg, qTaskInfo_t* pTaskInfo, uint64_t qId); * @param qId
* @return
*/
int32_t qCreateExecTask(void* tsdb, int32_t vgId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo);
/** /**
* the main query execution function, including query on both table and multiple tables, * The main task execution function, including query on both table and multiple tables,
* which are decided according to the tag or table name query conditions * which are decided according to the tag or table name query conditions
* *
* @param qinfo * @param tinfo
* @param handle
* @return * @return
*/ */
bool qExecTask(qTaskInfo_t qinfo, uint64_t *qId); int32_t qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle);
/** /**
* Retrieve the produced results information, if current query is not paused or completed, * Retrieve the produced results information, if current query is not paused or completed,
...@@ -60,7 +66,7 @@ int32_t qRetrieveQueryResultInfo(qTaskInfo_t qinfo, bool* buildRes, void* pRspCo ...@@ -60,7 +66,7 @@ int32_t qRetrieveQueryResultInfo(qTaskInfo_t qinfo, bool* buildRes, void* pRspCo
* @param contLen payload length * @param contLen payload length
* @return * @return
*/ */
int32_t qDumpRetrieveResult(qTaskInfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec); //int32_t qDumpRetrieveResult(qTaskInfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec);
/** /**
* return the transporter context (RPC) * return the transporter context (RPC)
...@@ -81,7 +87,7 @@ int32_t qKillTask(qTaskInfo_t qinfo); ...@@ -81,7 +87,7 @@ int32_t qKillTask(qTaskInfo_t qinfo);
* @param qinfo * @param qinfo
* @return * @return
*/ */
int32_t qIsQueryCompleted(qTaskInfo_t qinfo); int32_t qIsTaskCompleted(qTaskInfo_t qinfo);
/** /**
* destroy query info structure * destroy query info structure
...@@ -113,7 +119,7 @@ int32_t qGetQualifiedTableIdList(void* pTableList, const char* tagCond, int32_t ...@@ -113,7 +119,7 @@ int32_t qGetQualifiedTableIdList(void* pTableList, const char* tagCond, int32_t
* @param numOfIndex * @param numOfIndex
* @return * @return
*/ */
int32_t qCreateTableGroupByGroupExpr(SArray* pTableIdList, TSKEY skey, STableGroupInfo groupInfo, SColIndex* groupByIndex, int32_t numOfIndex); //int32_t qCreateTableGroupByGroupExpr(SArray* pTableIdList, TSKEY skey, STableGroupInfo groupInfo, SColIndex* groupByIndex, int32_t numOfIndex);
/** /**
* Update the table id list of a given query. * Update the table id list of a given query.
......
include/libs/scheduler/scheduler.h
浏览文件 @ 369b5559
...@@ -20,8 +20,10 @@ ...@@ -20,8 +20,10 @@
extern "C" { extern "C" {
#endif #endif
#include "planner.h"
#include "catalog.h" #include "catalog.h"
#include "planner.h"
struct SSchJob;
typedef struct SSchedulerCfg { typedef struct SSchedulerCfg {
uint32_t maxJobNum; uint32_t maxJobNum;
...@@ -65,7 +67,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg); ...@@ -65,7 +67,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg);
* @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr
* @return * @return
*/ */
int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob, SQueryResult *pRes); int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes);
/** /**
* Process the query job, generated according to the query physical plan. * Process the query job, generated according to the query physical plan.
...@@ -73,7 +75,7 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void ...@@ -73,7 +75,7 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void
* @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr
* @return * @return
*/ */
int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob); int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob);
/** /**
* Fetch query result from the remote query executor * Fetch query result from the remote query executor
...@@ -81,7 +83,7 @@ int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, ...@@ -81,7 +83,7 @@ int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag,
* @param data * @param data
* @return * @return
*/ */
int32_t scheduleFetchRows(void *pJob, void **data); int32_t scheduleFetchRows(struct SSchJob *pJob, void **data);
/** /**
...@@ -89,7 +91,7 @@ int32_t scheduleFetchRows(void *pJob, void **data); ...@@ -89,7 +91,7 @@ int32_t scheduleFetchRows(void *pJob, void **data);
* @param pJob * @param pJob
* @return * @return
*/ */
int32_t scheduleCancelJob(void *pJob); //int32_t scheduleCancelJob(void *pJob);
/** /**
* Free the query job * Free the query job
......
source/client/inc/clientInt.h
浏览文件 @ 369b5559
...@@ -113,6 +113,7 @@ typedef struct SRequestSendRecvBody { ...@@ -113,6 +113,7 @@ typedef struct SRequestSendRecvBody {
tsem_t rspSem; // not used now tsem_t rspSem; // not used now
void* fp; void* fp;
SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed. SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed.
struct SSchJob *pQueryJob; // query job, created according to sql query DAG.
SDataBuf requestMsg; SDataBuf requestMsg;
SReqResultInfo resInfo; SReqResultInfo resInfo;
} SRequestSendRecvBody; } SRequestSendRecvBody;
...@@ -129,7 +130,7 @@ typedef struct SRequestObj { ...@@ -129,7 +130,7 @@ typedef struct SRequestObj {
char *msgBuf; char *msgBuf;
void *pInfo; // sql parse info, generated by parser module void *pInfo; // sql parse info, generated by parser module
int32_t code; int32_t code;
uint64_t affectedRows; uint64_t affectedRows; // todo remove it
SQueryExecMetric metric; SQueryExecMetric metric;
SRequestSendRecvBody body; SRequestSendRecvBody body;
} SRequestObj; } SRequestObj;
...@@ -161,12 +162,13 @@ int taos_options_imp(TSDB_OPTION option, const char *str); ...@@ -161,12 +162,13 @@ int taos_options_imp(TSDB_OPTION option, const char *str);
void* openTransporter(const char *user, const char *auth, int32_t numOfThreads); void* openTransporter(const char *user, const char *auth, int32_t numOfThreads);
void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet);
void initMsgHandleFp(); void initMsgHandleFp();
TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port); TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port);
TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen);
void *doFetchRow(SRequestObj* pRequest); void *doFetchRow(SRequestObj* pRequest);
void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows);
#ifdef __cplusplus #ifdef __cplusplus
......
source/client/src/clientImpl.c
浏览文件 @ 369b5559
#include "../../libs/scheduler/inc/schedulerInt.h"
#include "clientInt.h" #include "clientInt.h"
#include "clientLog.h" #include "clientLog.h"
#include "parser.h"
#include "planner.h"
#include "scheduler.h"
#include "tdef.h" #include "tdef.h"
#include "tep.h" #include "tep.h"
#include "tglobal.h" #include "tglobal.h"
...@@ -8,9 +12,6 @@ ...@@ -8,9 +12,6 @@
#include "tnote.h" #include "tnote.h"
#include "tpagedfile.h" #include "tpagedfile.h"
#include "tref.h" #include "tref.h"
#include "parser.h"
#include "planner.h"
#include "scheduler.h"
#define CHECK_CODE_GOTO(expr, lable) \ #define CHECK_CODE_GOTO(expr, lable) \
do { \ do { \
...@@ -57,6 +58,7 @@ static char* getClusterKey(const char* user, const char* auth, const char* ip, i ...@@ -57,6 +58,7 @@ static char* getClusterKey(const char* user, const char* auth, const char* ip, i
} }
static STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo); static STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo);
static void setResSchemaInfo(SReqResultInfo* pResInfo, const SDataBlockSchema* pDataBlockSchema);
TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port) { TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port) {
if (taos_init() != TSDB_CODE_SUCCESS) { if (taos_init() != TSDB_CODE_SUCCESS) {
...@@ -197,19 +199,49 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) { ...@@ -197,19 +199,49 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) {
int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag) { int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag) {
pRequest->type = pQueryNode->type; pRequest->type = pQueryNode->type;
return qCreateQueryDag(pQueryNode, pDag, pRequest->requestId);
SReqResultInfo* pResInfo = &pRequest->body.resInfo;
int32_t code = qCreateQueryDag(pQueryNode, pDag, pRequest->requestId);
if (code != 0) {
return code;
}
if (pQueryNode->type == TSDB_SQL_SELECT) {
SArray* pa = taosArrayGetP((*pDag)->pSubplans, 0);
SSubplan* pPlan = taosArrayGetP(pa, 0);
SDataBlockSchema* pDataBlockSchema = &(pPlan->pDataSink->schema);
setResSchemaInfo(pResInfo, pDataBlockSchema);
pRequest->type = TDMT_VND_QUERY;
}
return code;
}
void setResSchemaInfo(SReqResultInfo* pResInfo, const SDataBlockSchema* pDataBlockSchema) {
assert(pDataBlockSchema != NULL && pDataBlockSchema->numOfCols > 0);
pResInfo->numOfCols = pDataBlockSchema->numOfCols;
pResInfo->fields = calloc(pDataBlockSchema->numOfCols, sizeof(pDataBlockSchema->pSchema[0]));
for (int32_t i = 0; i < pResInfo->numOfCols; ++i) {
SSchema* pSchema = &pDataBlockSchema->pSchema[i];
pResInfo->fields[i].bytes = pSchema->bytes;
pResInfo->fields[i].type = pSchema->type;
tstrncpy(pResInfo->fields[i].name, pSchema[i].name, tListLen(pResInfo->fields[i].name));
}
} }
int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, void** pJob) { int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) {
if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) { if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) {
SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf};
int32_t code = scheduleExecJob(pRequest->pTscObj->pTransporter, NULL, pDag, pJob, &res); int32_t code = scheduleExecJob(pRequest->pTscObj->pTransporter, NULL, pDag, &pRequest->body.pQueryJob, &res);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
// handle error and retry // handle error and retry
} else { } else {
if (*pJob != NULL) { if (pRequest->body.pQueryJob != NULL) {
scheduleFreeJob(*pJob); scheduleFreeJob(pRequest->body.pQueryJob);
} }
} }
...@@ -217,7 +249,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, void** pJob) { ...@@ -217,7 +249,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, void** pJob) {
return res.code; return res.code;
} }
return scheduleAsyncExecJob(pRequest->pTscObj->pTransporter, NULL /*todo appInfo.xxx*/, pDag, pJob); return scheduleAsyncExecJob(pRequest->pTscObj->pTransporter, NULL, pDag, &pRequest->body.pQueryJob);
} }
TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen) { TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen) {
...@@ -294,7 +326,6 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { ...@@ -294,7 +326,6 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) {
SRequestObj *pRequest = NULL; SRequestObj *pRequest = NULL;
SQueryNode *pQuery = NULL; SQueryNode *pQuery = NULL;
SQueryDag *pDag = NULL; SQueryDag *pDag = NULL;
void *pJob = NULL;
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
...@@ -304,9 +335,8 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { ...@@ -304,9 +335,8 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) {
CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return); CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return);
} else { } else {
CHECK_CODE_GOTO(getPlan(pRequest, pQuery, &pDag), _return); CHECK_CODE_GOTO(getPlan(pRequest, pQuery, &pDag), _return);
CHECK_CODE_GOTO(scheduleQuery(pRequest, pDag, &pJob), _return); CHECK_CODE_GOTO(scheduleQuery(pRequest, pDag), _return);
pRequest->code = terrno; pRequest->code = terrno;
return pRequest;
} }
_return: _return:
...@@ -315,6 +345,7 @@ _return: ...@@ -315,6 +345,7 @@ _return:
if (NULL != pRequest && TSDB_CODE_SUCCESS != terrno) { if (NULL != pRequest && TSDB_CODE_SUCCESS != terrno) {
pRequest->code = terrno; pRequest->code = terrno;
} }
return pRequest; return pRequest;
} }
...@@ -513,7 +544,17 @@ void* doFetchRow(SRequestObj* pRequest) { ...@@ -513,7 +544,17 @@ void* doFetchRow(SRequestObj* pRequest) {
SReqResultInfo* pResultInfo = &pRequest->body.resInfo; SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) { if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) {
if (pRequest->type == TDMT_MND_SHOW) { if (pRequest->type == TDMT_VND_QUERY) {
pRequest->type = TDMT_VND_FETCH;
scheduleFetchRows(pRequest->body.pQueryJob, (void **)&pRequest->body.resInfo.pData);
pResultInfo->current = 0;
if (pResultInfo->numOfRows <= pResultInfo->current) {
return NULL;
}
goto _return;
} else if (pRequest->type == TDMT_MND_SHOW) {
pRequest->type = TDMT_MND_SHOW_RETRIEVE; pRequest->type = TDMT_MND_SHOW_RETRIEVE;
} else if (pRequest->type == TDMT_VND_SHOW_TABLES) { } else if (pRequest->type == TDMT_VND_SHOW_TABLES) {
pRequest->type = TDMT_VND_SHOW_TABLES_FETCH; pRequest->type = TDMT_VND_SHOW_TABLES_FETCH;
...@@ -556,6 +597,8 @@ void* doFetchRow(SRequestObj* pRequest) { ...@@ -556,6 +597,8 @@ void* doFetchRow(SRequestObj* pRequest) {
} }
} }
_return:
for(int32_t i = 0; i < pResultInfo->numOfCols; ++i) { for(int32_t i = 0; i < pResultInfo->numOfCols; ++i) {
pResultInfo->row[i] = pResultInfo->pCol[i] + pResultInfo->fields[i].bytes * pResultInfo->current; pResultInfo->row[i] = pResultInfo->pCol[i] + pResultInfo->fields[i].bytes * pResultInfo->current;
if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) {
......
source/client/test/clientTests.cpp
浏览文件 @ 369b5559
...@@ -24,7 +24,6 @@ ...@@ -24,7 +24,6 @@
#include "../inc/clientInt.h" #include "../inc/clientInt.h"
#include "taos.h" #include "taos.h"
#include "tglobal.h"
namespace { namespace {
void showDB(TAOS* pConn) { void showDB(TAOS* pConn) {
...@@ -57,449 +56,449 @@ TEST(testCase, connect_Test) { ...@@ -57,449 +56,449 @@ TEST(testCase, connect_Test) {
taos_close(pConn); taos_close(pConn);
} }
TEST(testCase, create_user_Test) { //TEST(testCase, create_user_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
assert(pConn != NULL); // assert(pConn != NULL);
//
TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); // TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'");
if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { // if (taos_errno(pRes) != TSDB_CODE_SUCCESS) {
printf("failed to create user, reason:%s\n", taos_errstr(pRes)); // printf("failed to create user, reason:%s\n", taos_errstr(pRes));
} // }
//
taos_free_result(pRes); // taos_free_result(pRes);
taos_close(pConn); // taos_close(pConn);
} //}
//
TEST(testCase, create_account_Test) { //TEST(testCase, create_account_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
assert(pConn != NULL); // assert(pConn != NULL);
//
TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); // TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'");
if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { // if (taos_errno(pRes) != TSDB_CODE_SUCCESS) {
printf("failed to create user, reason:%s\n", taos_errstr(pRes)); // printf("failed to create user, reason:%s\n", taos_errstr(pRes));
} // }
//
taos_free_result(pRes); // taos_free_result(pRes);
taos_close(pConn); // taos_close(pConn);
} //}
//
TEST(testCase, drop_account_Test) { //TEST(testCase, drop_account_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
assert(pConn != NULL); // assert(pConn != NULL);
//
TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); // TAOS_RES* pRes = taos_query(pConn, "drop account aabc");
if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { // if (taos_errno(pRes) != TSDB_CODE_SUCCESS) {
printf("failed to create user, reason:%s\n", taos_errstr(pRes)); // printf("failed to create user, reason:%s\n", taos_errstr(pRes));
} // }
//
taos_free_result(pRes); // taos_free_result(pRes);
taos_close(pConn); // taos_close(pConn);
} //}
//
TEST(testCase, show_user_Test) { //TEST(testCase, show_user_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
assert(pConn != NULL); // assert(pConn != NULL);
//
TAOS_RES* pRes = taos_query(pConn, "show users"); // TAOS_RES* pRes = taos_query(pConn, "show users");
TAOS_ROW pRow = NULL; // TAOS_ROW pRow = NULL;
//
TAOS_FIELD* pFields = taos_fetch_fields(pRes); // TAOS_FIELD* pFields = taos_fetch_fields(pRes);
int32_t numOfFields = taos_num_fields(pRes); // int32_t numOfFields = taos_num_fields(pRes);
//
char str[512] = {0}; // char str[512] = {0};
while ((pRow = taos_fetch_row(pRes)) != NULL) { // while ((pRow = taos_fetch_row(pRes)) != NULL) {
int32_t code = taos_print_row(str, pRow, pFields, numOfFields); // int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
printf("%s\n", str); // printf("%s\n", str);
} // }
//
taos_free_result(pRes); // taos_free_result(pRes);
taos_close(pConn); // taos_close(pConn);
} //}
//
TEST(testCase, drop_user_Test) { //TEST(testCase, drop_user_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
assert(pConn != NULL); // assert(pConn != NULL);
//
TAOS_RES* pRes = taos_query(pConn, "drop user abc"); // TAOS_RES* pRes = taos_query(pConn, "drop user abc");
if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { // if (taos_errno(pRes) != TSDB_CODE_SUCCESS) {
printf("failed to create user, reason:%s\n", taos_errstr(pRes)); // printf("failed to create user, reason:%s\n", taos_errstr(pRes));
} // }
//
taos_free_result(pRes); // taos_free_result(pRes);
taos_close(pConn); // taos_close(pConn);
} //}
//
TEST(testCase, show_db_Test) { //TEST(testCase, show_db_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
assert(pConn != NULL); // assert(pConn != NULL);
//
TAOS_RES* pRes = taos_query(pConn, "show databases"); // TAOS_RES* pRes = taos_query(pConn, "show databases");
TAOS_ROW pRow = NULL; // TAOS_ROW pRow = NULL;
//
TAOS_FIELD* pFields = taos_fetch_fields(pRes); // TAOS_FIELD* pFields = taos_fetch_fields(pRes);
int32_t numOfFields = taos_num_fields(pRes); // int32_t numOfFields = taos_num_fields(pRes);
//
char str[512] = {0}; // char str[512] = {0};
while ((pRow = taos_fetch_row(pRes)) != NULL) { // while ((pRow = taos_fetch_row(pRes)) != NULL) {
int32_t code = taos_print_row(str, pRow, pFields, numOfFields); // int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
printf("%s\n", str); // printf("%s\n", str);
} // }
//
taos_close(pConn); // taos_close(pConn);
} //}
//
TEST(testCase, create_db_Test) { //TEST(testCase, create_db_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
assert(pConn != NULL); // assert(pConn != NULL);
//
TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); // TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2");
if (taos_errno(pRes) != 0) { // if (taos_errno(pRes) != 0) {
printf("error in create db, reason:%s\n", taos_errstr(pRes)); // printf("error in create db, reason:%s\n", taos_errstr(pRes));
} // }
//
TAOS_FIELD* pFields = taos_fetch_fields(pRes); // TAOS_FIELD* pFields = taos_fetch_fields(pRes);
ASSERT_TRUE(pFields == NULL); // ASSERT_TRUE(pFields == NULL);
//
int32_t numOfFields = taos_num_fields(pRes); // int32_t numOfFields = taos_num_fields(pRes);
ASSERT_EQ(numOfFields, 0); // ASSERT_EQ(numOfFields, 0);
//
taos_free_result(pRes); // taos_free_result(pRes);
//
pRes = taos_query(pConn, "create database abc1 vgroups 4"); // pRes = taos_query(pConn, "create database abc1 vgroups 4");
if (taos_errno(pRes) != 0) { // if (taos_errno(pRes) != 0) {
printf("error in create db, reason:%s\n", taos_errstr(pRes)); // printf("error in create db, reason:%s\n", taos_errstr(pRes));
} // }
taos_close(pConn); // taos_close(pConn);
} //}
//
TEST(testCase, create_dnode_Test) { //TEST(testCase, create_dnode_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
assert(pConn != NULL); // assert(pConn != NULL);
//
TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); // TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000");
if (taos_errno(pRes) != 0) { // if (taos_errno(pRes) != 0) {
printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); // printf("error in create dnode, reason:%s\n", taos_errstr(pRes));
} // }
taos_free_result(pRes); // taos_free_result(pRes);
//
pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); // pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000");
if (taos_errno(pRes) != 0) { // if (taos_errno(pRes) != 0) {
printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); // printf("failed to create dnode, reason:%s\n", taos_errstr(pRes));
} // }
taos_free_result(pRes); // taos_free_result(pRes);
//
taos_close(pConn); // taos_close(pConn);
} //}
//
TEST(testCase, drop_dnode_Test) { //TEST(testCase, drop_dnode_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
assert(pConn != NULL); // assert(pConn != NULL);
//
TAOS_RES* pRes = taos_query(pConn, "drop dnode 2"); // TAOS_RES* pRes = taos_query(pConn, "drop dnode 2");
if (taos_errno(pRes) != 0) { // if (taos_errno(pRes) != 0) {
printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); // printf("error in drop dnode, reason:%s\n", taos_errstr(pRes));
} // }
//
TAOS_FIELD* pFields = taos_fetch_fields(pRes); // TAOS_FIELD* pFields = taos_fetch_fields(pRes);
ASSERT_TRUE(pFields == NULL); // ASSERT_TRUE(pFields == NULL);
//
int32_t numOfFields = taos_num_fields(pRes); // int32_t numOfFields = taos_num_fields(pRes);
ASSERT_EQ(numOfFields, 0); // ASSERT_EQ(numOfFields, 0);
//
taos_free_result(pRes); // taos_free_result(pRes);
taos_close(pConn); // taos_close(pConn);
} //}
//
TEST(testCase, use_db_test) { //TEST(testCase, use_db_test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
assert(pConn != NULL); // assert(pConn != NULL);
//
TAOS_RES* pRes = taos_query(pConn, "use abc1"); // TAOS_RES* pRes = taos_query(pConn, "use abc1");
if (taos_errno(pRes) != 0) { // if (taos_errno(pRes) != 0) {
printf("error in use db, reason:%s\n", taos_errstr(pRes)); // printf("error in use db, reason:%s\n", taos_errstr(pRes));
} // }
//
TAOS_FIELD* pFields = taos_fetch_fields(pRes); // TAOS_FIELD* pFields = taos_fetch_fields(pRes);
ASSERT_TRUE(pFields == NULL); // ASSERT_TRUE(pFields == NULL);
//
int32_t numOfFields = taos_num_fields(pRes); // int32_t numOfFields = taos_num_fields(pRes);
ASSERT_EQ(numOfFields, 0); // ASSERT_EQ(numOfFields, 0);
//
taos_close(pConn); // taos_close(pConn);
} //}
//
// TEST(testCase, drop_db_test) { //// TEST(testCase, drop_db_test) {
//// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
//// assert(pConn != NULL);
////
//// showDB(pConn);
////
//// TAOS_RES* pRes = taos_query(pConn, "drop database abc1");
//// if (taos_errno(pRes) != 0) {
//// printf("failed to drop db, reason:%s\n", taos_errstr(pRes));
//// }
//// taos_free_result(pRes);
////
//// showDB(pConn);
////
//// pRes = taos_query(pConn, "create database abc1");
//// if (taos_errno(pRes) != 0) {
//// printf("create to drop db, reason:%s\n", taos_errstr(pRes));
//// }
//// taos_free_result(pRes);
//// taos_close(pConn);
////}
//
//TEST(testCase, create_stable_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL);
//
// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2");
// if (taos_errno(pRes) != 0) {
// printf("error in create db, reason:%s\n", taos_errstr(pRes));
// }
// taos_free_result(pRes);
//
// pRes = taos_query(pConn, "use abc1");
// if (taos_errno(pRes) != 0) {
// printf("error in use db, reason:%s\n", taos_errstr(pRes));
// }
// taos_free_result(pRes);
//
// pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)");
// if (taos_errno(pRes) != 0) {
// printf("error in create stable, reason:%s\n", taos_errstr(pRes));
// }
//
// TAOS_FIELD* pFields = taos_fetch_fields(pRes);
// ASSERT_TRUE(pFields == NULL);
//
// int32_t numOfFields = taos_num_fields(pRes);
// ASSERT_EQ(numOfFields, 0);
//
// taos_free_result(pRes);
// taos_close(pConn);
//}
//
//TEST(testCase, create_table_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL);
//
// TAOS_RES* pRes = taos_query(pConn, "use abc1");
// taos_free_result(pRes);
//
// pRes = taos_query(pConn, "create table tm0(ts timestamp, k int)");
// taos_free_result(pRes);
//
// taos_close(pConn);
//}
//
//TEST(testCase, create_ctable_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL);
//
// TAOS_RES* pRes = taos_query(pConn, "use abc1");
// if (taos_errno(pRes) != 0) {
// printf("failed to use db, reason:%s\n", taos_errstr(pRes));
// }
// taos_free_result(pRes);
//
// pRes = taos_query(pConn, "create table tm0 using st1 tags(1)");
// if (taos_errno(pRes) != 0) {
// printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes));
// }
//
// taos_free_result(pRes);
// taos_close(pConn);
//}
//
//TEST(testCase, show_stable_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL);
//
// TAOS_RES* pRes = taos_query(pConn, "use abc1");
// if (taos_errno(pRes) != 0) {
// printf("failed to use db, reason:%s\n", taos_errstr(pRes));
// }
// taos_free_result(pRes);
//
// pRes = taos_query(pConn, "show stables");
// if (taos_errno(pRes) != 0) {
// printf("failed to show stables, reason:%s\n", taos_errstr(pRes));
// taos_free_result(pRes);
// ASSERT_TRUE(false);
// }
//
// TAOS_ROW pRow = NULL;
// TAOS_FIELD* pFields = taos_fetch_fields(pRes);
// int32_t numOfFields = taos_num_fields(pRes);
//
// char str[512] = {0};
// while ((pRow = taos_fetch_row(pRes)) != NULL) {
// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
// printf("%s\n", str);
// }
//
// taos_free_result(pRes);
// taos_close(pConn);
//}
//
//TEST(testCase, show_vgroup_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL);
//
// TAOS_RES* pRes = taos_query(pConn, "use abc1");
// if (taos_errno(pRes) != 0) {
// printf("failed to use db, reason:%s\n", taos_errstr(pRes));
// }
// taos_free_result(pRes);
//
// pRes = taos_query(pConn, "show vgroups");
// if (taos_errno(pRes) != 0) {
// printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes));
// taos_free_result(pRes);
// ASSERT_TRUE(false);
// }
//
// TAOS_ROW pRow = NULL;
//
// TAOS_FIELD* pFields = taos_fetch_fields(pRes);
// int32_t numOfFields = taos_num_fields(pRes);
//
// char str[512] = {0};
// while ((pRow = taos_fetch_row(pRes)) != NULL) {
// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
// printf("%s\n", str);
// }
//
// taos_free_result(pRes);
// taos_close(pConn);
//}
//
//TEST(testCase, create_multiple_tables) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// ASSERT_NE(pConn, nullptr);
//
// TAOS_RES* pRes = taos_query(pConn, "use abc1");
// if (taos_errno(pRes) != 0) {
// printf("failed to use db, reason:%s\n", taos_errstr(pRes));
// taos_free_result(pRes);
// taos_close(pConn);
// return;
// }
//
// taos_free_result(pRes);
//
// pRes = taos_query(pConn, "create table t_2 using st1 tags(1)");
// if (taos_errno(pRes) != 0) {
// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes));
// taos_free_result(pRes);
// ASSERT_TRUE(false);
// }
//
// taos_free_result(pRes);
// pRes = taos_query(pConn, "create table t_3 using st1 tags(2)");
// if (taos_errno(pRes) != 0) {
// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes));
// taos_free_result(pRes);
// ASSERT_TRUE(false);
// }
//
// TAOS_ROW pRow = NULL;
// TAOS_FIELD* pFields = taos_fetch_fields(pRes);
// int32_t numOfFields = taos_num_fields(pRes);
//
// char str[512] = {0};
// while ((pRow = taos_fetch_row(pRes)) != NULL) {
// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
// printf("%s\n", str);
// }
//
// taos_free_result(pRes);
//
// for (int32_t i = 0; i < 20; ++i) {
// char sql[512] = {0};
// snprintf(sql, tListLen(sql),
// "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i,
// (i + 1) * 30, (i + 2) * 40);
// TAOS_RES* pres = taos_query(pConn, sql);
// if (taos_errno(pres) != 0) {
// printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres));
// }
// taos_free_result(pres);
// }
//
// taos_close(pConn);
//}
//
//TEST(testCase, show_table_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL); // assert(pConn != NULL);
// //
// showDB(pConn); // TAOS_RES* pRes = taos_query(pConn, "use abc1");
// taos_free_result(pRes);
//
// pRes = taos_query(pConn, "show tables");
// if (taos_errno(pRes) != 0) {
// printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes));
// taos_free_result(pRes);
// ASSERT_TRUE(false);
// }
//
// TAOS_ROW pRow = NULL;
// TAOS_FIELD* pFields = taos_fetch_fields(pRes);
// int32_t numOfFields = taos_num_fields(pRes);
//
// char str[512] = {0};
// while ((pRow = taos_fetch_row(pRes)) != NULL) {
// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
// printf("%s\n", str);
// }
//
// taos_free_result(pRes);
// taos_close(pConn);
//}
//
//TEST(testCase, drop_stable_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL);
// //
// TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); // TAOS_RES* pRes = taos_query(pConn, "create database abc1");
// if (taos_errno(pRes) != 0) { // if (taos_errno(pRes) != 0) {
// printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); // printf("error in creating db, reason:%s\n", taos_errstr(pRes));
// } // }
// taos_free_result(pRes); // taos_free_result(pRes);
// //
// showDB(pConn); // pRes = taos_query(pConn, "use abc1");
// if (taos_errno(pRes) != 0) {
// printf("error in using db, reason:%s\n", taos_errstr(pRes));
// }
// taos_free_result(pRes);
// //
// pRes = taos_query(pConn, "create database abc1"); // pRes = taos_query(pConn, "drop stable st1");
// if (taos_errno(pRes) != 0) { // if (taos_errno(pRes) != 0) {
// printf("create to drop db, reason:%s\n", taos_errstr(pRes)); // printf("failed to drop stable, reason:%s\n", taos_errstr(pRes));
// } // }
//
// taos_free_result(pRes); // taos_free_result(pRes);
// taos_close(pConn); // taos_close(pConn);
//} //}
//
TEST(testCase, create_stable_Test) { //TEST(testCase, generated_request_id_test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
assert(pConn != NULL); //
// for (int32_t i = 0; i < 50000; ++i) {
TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); // uint64_t v = generateRequestId();
if (taos_errno(pRes) != 0) { // void* result = taosHashGet(phash, &v, sizeof(v));
printf("error in create db, reason:%s\n", taos_errstr(pRes)); // if (result != nullptr) {
} // printf("0x%lx, index:%d\n", v, i);
taos_free_result(pRes); // }
// assert(result == nullptr);
pRes = taos_query(pConn, "use abc1"); // taosHashPut(phash, &v, sizeof(v), NULL, 0);
if (taos_errno(pRes) != 0) { // }
printf("error in use db, reason:%s\n", taos_errstr(pRes)); //
} // taosHashCleanup(phash);
taos_free_result(pRes); //}
pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)");
if (taos_errno(pRes) != 0) {
printf("error in create stable, reason:%s\n", taos_errstr(pRes));
}
TAOS_FIELD* pFields = taos_fetch_fields(pRes);
ASSERT_TRUE(pFields == NULL);
int32_t numOfFields = taos_num_fields(pRes);
ASSERT_EQ(numOfFields, 0);
taos_free_result(pRes);
taos_close(pConn);
}
TEST(testCase, create_table_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "use abc1");
taos_free_result(pRes);
pRes = taos_query(pConn, "create table tm0(ts timestamp, k int)");
taos_free_result(pRes);
taos_close(pConn);
}
TEST(testCase, create_ctable_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "use abc1");
if (taos_errno(pRes) != 0) {
printf("failed to use db, reason:%s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
pRes = taos_query(pConn, "create table tm0 using st1 tags(1)");
if (taos_errno(pRes) != 0) {
printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
taos_close(pConn);
}
TEST(testCase, show_stable_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "use abc1");
if (taos_errno(pRes) != 0) {
printf("failed to use db, reason:%s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
pRes = taos_query(pConn, "show stables");
if (taos_errno(pRes) != 0) {
printf("failed to show stables, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes);
ASSERT_TRUE(false);
}
TAOS_ROW pRow = NULL;
TAOS_FIELD* pFields = taos_fetch_fields(pRes);
int32_t numOfFields = taos_num_fields(pRes);
char str[512] = {0};
while ((pRow = taos_fetch_row(pRes)) != NULL) {
int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
printf("%s\n", str);
}
taos_free_result(pRes);
taos_close(pConn);
}
TEST(testCase, show_vgroup_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "use abc1");
if (taos_errno(pRes) != 0) {
printf("failed to use db, reason:%s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
pRes = taos_query(pConn, "show vgroups");
if (taos_errno(pRes) != 0) {
printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes);
ASSERT_TRUE(false);
}
TAOS_ROW pRow = NULL;
TAOS_FIELD* pFields = taos_fetch_fields(pRes);
int32_t numOfFields = taos_num_fields(pRes);
char str[512] = {0};
while ((pRow = taos_fetch_row(pRes)) != NULL) {
int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
printf("%s\n", str);
}
taos_free_result(pRes);
taos_close(pConn);
}
TEST(testCase, create_multiple_tables) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr);
TAOS_RES* pRes = taos_query(pConn, "use abc1");
if (taos_errno(pRes) != 0) {
printf("failed to use db, reason:%s", taos_errstr(pRes));
taos_free_result(pRes);
taos_close(pConn);
return;
}
taos_free_result(pRes);
pRes = taos_query(pConn, "create table t_2 using st1 tags(1)");
if (taos_errno(pRes) != 0) {
printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes);
ASSERT_TRUE(false);
}
taos_free_result(pRes);
pRes = taos_query(pConn, "create table t_3 using st1 tags(2)");
if (taos_errno(pRes) != 0) {
printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes);
ASSERT_TRUE(false);
}
TAOS_ROW pRow = NULL;
TAOS_FIELD* pFields = taos_fetch_fields(pRes);
int32_t numOfFields = taos_num_fields(pRes);
char str[512] = {0};
while ((pRow = taos_fetch_row(pRes)) != NULL) {
int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
printf("%s\n", str);
}
taos_free_result(pRes);
for (int32_t i = 0; i < 20; ++i) {
char sql[512] = {0};
snprintf(sql, tListLen(sql),
"create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i,
(i + 1) * 30, (i + 2) * 40);
TAOS_RES* pres = taos_query(pConn, sql);
if (taos_errno(pres) != 0) {
printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres));
}
taos_free_result(pres);
}
taos_close(pConn);
}
TEST(testCase, show_table_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "use abc1");
taos_free_result(pRes);
pRes = taos_query(pConn, "show tables");
if (taos_errno(pRes) != 0) {
printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes);
ASSERT_TRUE(false);
}
TAOS_ROW pRow = NULL;
TAOS_FIELD* pFields = taos_fetch_fields(pRes);
int32_t numOfFields = taos_num_fields(pRes);
char str[512] = {0};
while ((pRow = taos_fetch_row(pRes)) != NULL) {
int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
printf("%s\n", str);
}
taos_free_result(pRes);
taos_close(pConn);
}
TEST(testCase, drop_stable_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "create database abc1");
if (taos_errno(pRes) != 0) {
printf("error in creating db, reason:%s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
pRes = taos_query(pConn, "use abc1");
if (taos_errno(pRes) != 0) {
printf("error in using db, reason:%s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
pRes = taos_query(pConn, "drop stable st1");
if (taos_errno(pRes) != 0) {
printf("failed to drop stable, reason:%s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
taos_close(pConn);
}
TEST(testCase, generated_request_id_test) {
SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
for (int32_t i = 0; i < 50000; ++i) {
uint64_t v = generateRequestId();
void* result = taosHashGet(phash, &v, sizeof(v));
if (result != nullptr) {
printf("0x%lx, index:%d\n", v, i);
}
assert(result == nullptr);
taosHashPut(phash, &v, sizeof(v), NULL, 0);
}
taosHashCleanup(phash);
}
// TEST(testCase, create_topic_Test) { // TEST(testCase, create_topic_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
...@@ -557,18 +556,29 @@ TEST(testCase, projection_query_tables) { ...@@ -557,18 +556,29 @@ TEST(testCase, projection_query_tables) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr); ASSERT_NE(pConn, nullptr);
TAOS_RES* pRes = taos_query(pConn, "use test1"); // TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2");
if (taos_errno(pRes) != 0) { // if (taos_errno(pRes) != 0) {
printf("failed to use db, reason:%s", taos_errstr(pRes)); // printf("failed to use db, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes); // taos_free_result(pRes);
return; // return;
} // }
// taos_free_result(pRes);
TAOS_RES* pRes = taos_query(pConn, "use abc1");
// pRes = taos_query(pConn, "create table m1 (ts timestamp, k int) tags(a int)");
taos_free_result(pRes); taos_free_result(pRes);
//
// pRes = taos_query(pConn, "create table tu using m1 tags(1)");
// taos_free_result(pRes);
//
// pRes = taos_query(pConn, "insert into tu values(now, 1)");
// taos_free_result(pRes);
pRes = taos_query(pConn, "select * from tm0"); pRes = taos_query(pConn, "select * from tu");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes); taos_free_result(pRes);
ASSERT_TRUE(false); ASSERT_TRUE(false);
} }
......
source/dnode/vnode/impl/src/vnodeQuery.c
浏览文件 @ 369b5559
...@@ -23,7 +23,7 @@ int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NULL, &pVnode->pQuery); ...@@ -23,7 +23,7 @@ int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NULL, &pVnode->pQuery);
int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
vTrace("query message is processed"); vTrace("query message is processed");
return qWorkerProcessQueryMsg(pVnode, pVnode->pQuery, pMsg); return qWorkerProcessQueryMsg(pVnode->pTsdb, pVnode->pQuery, pMsg);
} }
int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
......
source/dnode/vnode/tsdb/CMakeLists.txt
浏览文件 @ 369b5559
...@@ -13,6 +13,7 @@ else(0) ...@@ -13,6 +13,7 @@ else(0)
"src/tsdbReadImpl.c" "src/tsdbReadImpl.c"
"src/tsdbFile.c" "src/tsdbFile.c"
"src/tsdbFS.c" "src/tsdbFS.c"
"src/tsdbRead.c"
) )
endif(0) endif(0)
......
source/dnode/vnode/tsdb/src/tsdbCommit.c
浏览文件 @ 369b5559
...@@ -1253,7 +1253,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols * ...@@ -1253,7 +1253,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols *
pBlock->keyFirst = dataColsKeyFirst(pDataCols); pBlock->keyFirst = dataColsKeyFirst(pDataCols);
pBlock->keyLast = dataColsKeyLast(pDataCols); pBlock->keyLast = dataColsKeyLast(pDataCols);
tsdbDebug("vgId:%d tid:%d a block of data is written to file %s, offset %" PRId64 tsdbDebug("vgId:%d uid:%"PRId64" a block of data is written to file %s, offset %" PRId64
" numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64,
REPO_ID(pRepo), TABLE_TID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len, REPO_ID(pRepo), TABLE_TID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len,
pBlock->numOfCols, pBlock->keyFirst, pBlock->keyLast); pBlock->numOfCols, pBlock->keyFirst, pBlock->keyLast);
......
source/dnode/vnode/tsdb/src/tsdbRead.c
浏览文件 @ 369b5559
...@@ -13,18 +13,23 @@ ...@@ -13,18 +13,23 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "tsdb.h"
#include "tsdbDef.h"
#include "tsdbFS.h"
#include "tsdbLog.h"
#include "tsdbReadImpl.h"
#include "ttime.h"
#include "exception.h"
#include "os.h" #include "os.h"
#include "tdataformat.h"
#include "tskiplist.h"
#include "tulog.h"
#include "talgo.h" #include "talgo.h"
#include "tcompare.h" #include "tcompare.h"
#include "exception.h" #include "tdataformat.h"
#include "tskiplist.h"
#include "taosdef.h" #include "taosdef.h"
#include "tlosertree.h" #include "tlosertree.h"
#include "tsdbint.h" #include "tsdbint.h"
#include "texpr.h" #include "tmsg.h"
#define EXTRA_BYTES 2 #define EXTRA_BYTES 2
#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC)
...@@ -34,8 +39,7 @@ ...@@ -34,8 +39,7 @@
((SDataBlockInfo){.window = {.skey = (_block)->keyFirst, .ekey = (_block)->keyLast}, \ ((SDataBlockInfo){.window = {.skey = (_block)->keyFirst, .ekey = (_block)->keyLast}, \
.numOfCols = (_block)->numOfCols, \ .numOfCols = (_block)->numOfCols, \
.rows = (_block)->numOfRows, \ .rows = (_block)->numOfRows, \
.tid = (_checkInfo)->tableId.tid, \ .uid = (_checkInfo)->tableId})
.uid = (_checkInfo)->tableId.uid})
enum { enum {
TSDB_QUERY_TYPE_ALL = 1, TSDB_QUERY_TYPE_ALL = 1,
...@@ -62,7 +66,7 @@ typedef struct SQueryFilePos { ...@@ -62,7 +66,7 @@ typedef struct SQueryFilePos {
typedef struct SDataBlockLoadInfo { typedef struct SDataBlockLoadInfo {
SDFileSet* fileGroup; SDFileSet* fileGroup;
int32_t slot; int32_t slot;
int32_t tid; uint64_t uid;
SArray* pLoadedCols; SArray* pLoadedCols;
} SDataBlockLoadInfo; } SDataBlockLoadInfo;
...@@ -79,13 +83,13 @@ enum { ...@@ -79,13 +83,13 @@ enum {
typedef struct STableCheckInfo { typedef struct STableCheckInfo {
STableId tableId; uint64_t tableId;
TSKEY lastKey; TSKEY lastKey;
STable* pTableObj; STable* pTableObj;
SBlockInfo* pCompInfo; SBlockInfo* pCompInfo;
int32_t compSize; int32_t compSize;
int32_t numOfBlocks:29; // number of qualified data blocks not the original blocks int32_t numOfBlocks:29; // number of qualified data blocks not the original blocks
uint8_t chosen:2; // indicate which iterator should move forward uint8_t chosen:2; // indicate which iterator should move forward
bool initBuf; // whether to initialize the in-memory skip list iterator or not bool initBuf; // whether to initialize the in-memory skip list iterator or not
SSkipListIterator* iter; // mem buffer skip list iterator SSkipListIterator* iter; // mem buffer skip list iterator
SSkipListIterator* iiter; // imem buffer skip list iterator SSkipListIterator* iiter; // imem buffer skip list iterator
...@@ -111,8 +115,8 @@ typedef struct SIOCostSummary { ...@@ -111,8 +115,8 @@ typedef struct SIOCostSummary {
int64_t headFileLoadTime; int64_t headFileLoadTime;
} SIOCostSummary; } SIOCostSummary;
typedef struct STsdbQueryHandle { typedef struct STsdbReadHandle {
STsdbRepo* pTsdb; STsdb* pTsdb;
SQueryFilePos cur; // current position SQueryFilePos cur; // current position
int16_t order; int16_t order;
STimeWindow window; // the primary query time window that applies to all queries STimeWindow window; // the primary query time window that applies to all queries
...@@ -137,7 +141,8 @@ typedef struct STsdbQueryHandle { ...@@ -137,7 +141,8 @@ typedef struct STsdbQueryHandle {
STableBlockInfo* pDataBlockInfo; STableBlockInfo* pDataBlockInfo;
SDataCols *pDataCols; // in order to hold current file data block SDataCols *pDataCols; // in order to hold current file data block
int32_t allocSize; // allocated data block size int32_t allocSize; // allocated data block size
SMemRef *pMemRef; // STsdb
// STsdbMemTable * pMemTable;
SArray *defaultLoadColumn;// default load column SArray *defaultLoadColumn;// default load column
SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */
SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQueryAttr */ SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQueryAttr */
...@@ -145,7 +150,7 @@ typedef struct STsdbQueryHandle { ...@@ -145,7 +150,7 @@ typedef struct STsdbQueryHandle {
SArray *prev; // previous row which is before than time window SArray *prev; // previous row which is before than time window
SArray *next; // next row which is after the query time window SArray *next; // next row which is after the query time window
SIOCostSummary cost; SIOCostSummary cost;
} STsdbQueryHandle; } STsdbReadHandle;
typedef struct STableGroupSupporter { typedef struct STableGroupSupporter {
int32_t numOfCols; int32_t numOfCols;
...@@ -154,23 +159,23 @@ typedef struct STableGroupSupporter { ...@@ -154,23 +159,23 @@ typedef struct STableGroupSupporter {
} STableGroupSupporter; } STableGroupSupporter;
static STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList); static STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList);
static int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList); static int32_t checkForCachedLastRow(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo *groupList);
static int32_t checkForCachedLast(STsdbQueryHandle* pQueryHandle); static int32_t checkForCachedLast(STsdbReadHandle* pTsdbReadHandle);
static int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey); //static int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey);
static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); static void changeQueryHandleForInterpQuery(tsdbReadHandleT pHandle);
static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock); static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock);
static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order);
static int32_t tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, STsdbQueryHandle* pQueryHandle); static int32_t tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, STsdbReadHandle* pTsdbReadHandle);
static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2); static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2);
static int32_t doGetExternalRow(STsdbQueryHandle* pQueryHandle, int16_t type, SMemRef* pMemRef); //static int32_t doGetExternalRow(STsdbReadHandle* pTsdbReadHandle, int16_t type, void* pMemRef);
static void* doFreeColumnInfoData(SArray* pColumnInfoData); //static void* doFreeColumnInfoData(SArray* pColumnInfoData);
static void* destroyTableCheckInfo(SArray* pTableCheckInfo); //static void* destroyTableCheckInfo(SArray* pTableCheckInfo);
static bool tsdbGetExternalRow(TsdbQueryHandleT pHandle); static bool tsdbGetExternalRow(tsdbReadHandleT pHandle);
static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) {
pBlockLoadInfo->slot = -1; pBlockLoadInfo->slot = -1;
pBlockLoadInfo->tid = -1; pBlockLoadInfo->uid = 0;
pBlockLoadInfo->fileGroup = NULL; pBlockLoadInfo->fileGroup = NULL;
} }
...@@ -179,21 +184,21 @@ static void tsdbInitCompBlockLoadInfo(SLoadCompBlockInfo* pCompBlockLoadInfo) { ...@@ -179,21 +184,21 @@ static void tsdbInitCompBlockLoadInfo(SLoadCompBlockInfo* pCompBlockLoadInfo) {
pCompBlockLoadInfo->fileId = -1; pCompBlockLoadInfo->fileId = -1;
} }
static SArray* getColumnIdList(STsdbQueryHandle* pQueryHandle) { static SArray* getColumnIdList(STsdbReadHandle* pTsdbReadHandle) {
size_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); size_t numOfCols = QH_GET_NUM_OF_COLS(pTsdbReadHandle);
assert(numOfCols <= TSDB_MAX_COLUMNS); assert(numOfCols <= TSDB_MAX_COLUMNS);
SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t)); SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t));
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pColumns, i);
taosArrayPush(pIdList, &pCol->info.colId); taosArrayPush(pIdList, &pCol->info.colId);
} }
return pIdList; return pIdList;
} }
static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS) { static SArray* getDefaultLoadColumns(STsdbReadHandle* pTsdbReadHandle, bool loadTS) {
SArray* pLocalIdList = getColumnIdList(pQueryHandle); SArray* pLocalIdList = getColumnIdList(pTsdbReadHandle);
// check if the primary time stamp column needs to load // check if the primary time stamp column needs to load
int16_t colId = *(int16_t*)taosArrayGet(pLocalIdList, 0); int16_t colId = *(int16_t*)taosArrayGet(pLocalIdList, 0);
...@@ -207,63 +212,63 @@ static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS ...@@ -207,63 +212,63 @@ static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS
return pLocalIdList; return pLocalIdList;
} }
static void tsdbMayTakeMemSnapshot(STsdbQueryHandle* pQueryHandle, SArray* psTable) { static void tsdbMayTakeMemSnapshot(STsdbReadHandle* pTsdbReadHandle, SArray* psTable) {
assert(pQueryHandle != NULL && pQueryHandle->pMemRef != NULL); // assert(pTsdbReadHandle != NULL && pTsdbReadHandle->pMemRef != NULL);
//
SMemRef* pMemRef = pQueryHandle->pMemRef; // STsdbMemTable* pMemRef = pTsdbReadHandle->pMemRef;
if (pQueryHandle->pMemRef->ref++ == 0) { // if (pTsdbReadHandle->pMemRef->ref++ == 0) {
tsdbTakeMemSnapshot(pQueryHandle->pTsdb, &(pMemRef->snapshot), psTable); // tsdbTakeMemSnapshot(pTsdbReadHandle->pTsdb, &(pMemRef->snapshot), psTable);
} // }
//
taosArrayDestroy(psTable); // taosArrayDestroy(psTable);
}
static void tsdbMayUnTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) {
assert(pQueryHandle != NULL);
SMemRef* pMemRef = pQueryHandle->pMemRef;
if (pMemRef == NULL) { // it has been freed
return;
}
if (--pMemRef->ref == 0) {
tsdbUnTakeMemSnapShot(pQueryHandle->pTsdb, &(pMemRef->snapshot));
}
pQueryHandle->pMemRef = NULL;
} }
int64_t tsdbGetNumOfRowsInMemTable(TsdbQueryHandleT* pHandle) { static void tsdbMayUnTakeMemSnapshot(STsdbReadHandle* pTsdbReadHandle) {
STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; // assert(pTsdbReadHandle != NULL);
// STsdbMemTable* pMemRef = pTsdbReadHandle->pMemRef;
int64_t rows = 0; // if (pMemRef == NULL) { // it has been freed
SMemRef* pMemRef = pQueryHandle->pMemRef; // return;
if (pMemRef == NULL) { return rows; } // }
//
STableData* pMem = NULL; // if (--pMemRef->ref == 0) {
STableData* pIMem = NULL; // tsdbUnTakeMemSnapShot(pTsdbReadHandle->pTsdb, &(pMemRef->snapshot));
// }
SMemTable* pMemT = pMemRef->snapshot.mem; //
SMemTable* pIMemT = pMemRef->snapshot.imem; // pTsdbReadHandle->pMemRef = NULL;
size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
for (int32_t i = 0; i < size; ++i) {
STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
if (pMemT && pCheckInfo->tableId.tid < pMemT->maxTables) {
pMem = pMemT->tData[pCheckInfo->tableId.tid];
rows += (pMem && pMem->uid == pCheckInfo->tableId.uid) ? pMem->numOfRows : 0;
}
if (pIMemT && pCheckInfo->tableId.tid < pIMemT->maxTables) {
pIMem = pIMemT->tData[pCheckInfo->tableId.tid];
rows += (pIMem && pIMem->uid == pCheckInfo->tableId.uid) ? pIMem->numOfRows : 0;
}
}
return rows;
} }
static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STableGroupInfo* pGroupList, STsdbMeta* pMeta, SArray** psTable) { //int64_t tsdbGetNumOfRowsInMemTable(tsdbReadHandleT* pHandle) {
// STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle;
//
// int64_t rows = 0;
// STsdbMemTable* pMemTable = pTsdbReadHandle->pMemTable;
// if (pMemTable == NULL) { return rows; }
//
//// STableData* pMem = NULL;
//// STableData* pIMem = NULL;
//
//// SMemTable* pMemT = pMemRef->snapshot.mem;
//// SMemTable* pIMemT = pMemRef->snapshot.imem;
//
// size_t size = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo);
// for (int32_t i = 0; i < size; ++i) {
// STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i);
//
//// if (pMemT && pCheckInfo->tableId < pMemT->maxTables) {
//// pMem = pMemT->tData[pCheckInfo->tableId];
//// rows += (pMem && pMem->uid == pCheckInfo->tableId) ? pMem->numOfRows : 0;
//// }
//// if (pIMemT && pCheckInfo->tableId < pIMemT->maxTables) {
//// pIMem = pIMemT->tData[pCheckInfo->tableId];
//// rows += (pIMem && pIMem->uid == pCheckInfo->tableId) ? pIMem->numOfRows : 0;
//// }
// }
// return rows;
//}
static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo* pGroupList, SArray** psTable) {
size_t sizeOfGroup = taosArrayGetSize(pGroupList->pGroupList); size_t sizeOfGroup = taosArrayGetSize(pGroupList->pGroupList);
assert(sizeOfGroup >= 1 && pMeta != NULL); assert(sizeOfGroup >= 1);
// allocate buffer in order to load data blocks from file // allocate buffer in order to load data blocks from file
SArray* pTableCheckInfo = taosArrayInit(pGroupList->numOfTables, sizeof(STableCheckInfo)); SArray* pTableCheckInfo = taosArrayInit(pGroupList->numOfTables, sizeof(STableCheckInfo));
...@@ -288,32 +293,28 @@ static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STa ...@@ -288,32 +293,28 @@ static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STa
STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(group, j); STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(group, j);
STableCheckInfo info = { .lastKey = pKeyInfo->lastKey, .pTableObj = pKeyInfo->pTable }; STableCheckInfo info = { .lastKey = pKeyInfo->lastKey, .pTableObj = pKeyInfo->pTable };
assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE || // assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE ||
info.pTableObj->type == TSDB_CHILD_TABLE || info.pTableObj->type == TSDB_STREAM_TABLE)); // info.pTableObj->type == TSDB_CHILD_TABLE || info.pTableObj->type == TSDB_STREAM_TABLE));
info.tableId.tid = info.pTableObj->tableId.tid; info.tableId = pKeyInfo->uid;
info.tableId.uid = info.pTableObj->tableId.uid;
if (ASCENDING_TRAVERSE(pQueryHandle->order)) { if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
if (info.lastKey == INT64_MIN || info.lastKey < pQueryHandle->window.skey) { if (info.lastKey == INT64_MIN || info.lastKey < pTsdbReadHandle->window.skey) {
info.lastKey = pQueryHandle->window.skey; info.lastKey = pTsdbReadHandle->window.skey;
} }
assert(info.lastKey >= pQueryHandle->window.skey && info.lastKey <= pQueryHandle->window.ekey); assert(info.lastKey >= pTsdbReadHandle->window.skey && info.lastKey <= pTsdbReadHandle->window.ekey);
} else { } else {
assert(info.lastKey >= pQueryHandle->window.ekey && info.lastKey <= pQueryHandle->window.skey); assert(info.lastKey >= pTsdbReadHandle->window.ekey && info.lastKey <= pTsdbReadHandle->window.skey);
} }
taosArrayPush(pTableCheckInfo, &info); taosArrayPush(pTableCheckInfo, &info);
tsdbDebug("%p check table uid:%"PRId64", tid:%d from lastKey:%"PRId64" 0x%"PRIx64, pQueryHandle, info.tableId.uid, tsdbDebug("%p check table uid:%"PRId64" from lastKey:%"PRId64" 0x%"PRIx64, pTsdbReadHandle, info.tableId, info.lastKey, pTsdbReadHandle->qId);
info.tableId.tid, info.lastKey, pQueryHandle->qId);
} }
} }
taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar); // taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar);
size_t gsize = taosArrayGetSize(pTableCheckInfo); size_t gsize = taosArrayGetSize(pTableCheckInfo);
for (int32_t i = 0; i < gsize; ++i) { for (int32_t i = 0; i < gsize; ++i) {
STableCheckInfo* pInfo = (STableCheckInfo*) taosArrayGet(pTableCheckInfo, i); STableCheckInfo* pInfo = (STableCheckInfo*) taosArrayGet(pTableCheckInfo, i);
taosArrayPush(pTable, &pInfo->pTableObj); taosArrayPush(pTable, &pInfo->pTableObj);
...@@ -323,22 +324,22 @@ static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STa ...@@ -323,22 +324,22 @@ static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STa
return pTableCheckInfo; return pTableCheckInfo;
} }
static void resetCheckInfo(STsdbQueryHandle* pQueryHandle) { static void resetCheckInfo(STsdbReadHandle* pTsdbReadHandle) {
size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo);
assert(numOfTables >= 1); assert(numOfTables >= 1);
// todo apply the lastkey of table check to avoid to load header file // todo apply the lastkey of table check to avoid to load header file
for (int32_t i = 0; i < numOfTables; ++i) { for (int32_t i = 0; i < numOfTables; ++i) {
STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i); STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i);
pCheckInfo->lastKey = pQueryHandle->window.skey; pCheckInfo->lastKey = pTsdbReadHandle->window.skey;
pCheckInfo->iter = tSkipListDestroyIter(pCheckInfo->iter); pCheckInfo->iter = tSkipListDestroyIter(pCheckInfo->iter);
pCheckInfo->iiter = tSkipListDestroyIter(pCheckInfo->iiter); pCheckInfo->iiter = tSkipListDestroyIter(pCheckInfo->iiter);
pCheckInfo->initBuf = false; pCheckInfo->initBuf = false;
if (ASCENDING_TRAVERSE(pQueryHandle->order)) { if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
assert(pCheckInfo->lastKey >= pQueryHandle->window.skey); assert(pCheckInfo->lastKey >= pTsdbReadHandle->window.skey);
} else { } else {
assert(pCheckInfo->lastKey <= pQueryHandle->window.skey); assert(pCheckInfo->lastKey <= pTsdbReadHandle->window.skey);
} }
} }
} }
...@@ -358,38 +359,38 @@ static SArray* createCheckInfoFromCheckInfo(STableCheckInfo* pCheckInfo, TSKEY s ...@@ -358,38 +359,38 @@ static SArray* createCheckInfoFromCheckInfo(STableCheckInfo* pCheckInfo, TSKEY s
return pNew; return pNew;
} }
static bool emptyQueryTimewindow(STsdbQueryHandle* pQueryHandle) { static bool emptyQueryTimewindow(STsdbReadHandle* pTsdbReadHandle) {
assert(pQueryHandle != NULL); assert(pTsdbReadHandle != NULL);
STimeWindow* w = &pQueryHandle->window; STimeWindow* w = &pTsdbReadHandle->window;
bool asc = ASCENDING_TRAVERSE(pQueryHandle->order); bool asc = ASCENDING_TRAVERSE(pTsdbReadHandle->order);
return ((asc && w->skey > w->ekey) || (!asc && w->ekey > w->skey)); return ((asc && w->skey > w->ekey) || (!asc && w->ekey > w->skey));
} }
// Update the query time window according to the data time to live(TTL) information, in order to avoid to return // Update the query time window according to the data time to live(TTL) information, in order to avoid to return
// the expired data to client, even it is queried already. // the expired data to client, even it is queried already.
static int64_t getEarliestValidTimestamp(STsdbRepo* pTsdb) { static int64_t getEarliestValidTimestamp(STsdb* pTsdb) {
STsdbCfg* pCfg = &pTsdb->config; STsdbCfg* pCfg = &pTsdb->config;
int64_t now = taosGetTimestamp(pCfg->precision); int64_t now = taosGetTimestamp(pCfg->precision);
return now - (tsTickPerDay[pCfg->precision] * pCfg->keep) + 1; // needs to add one tick return now - (tsTickPerDay[pCfg->precision] * pCfg->keep) + 1; // needs to add one tick
} }
static void setQueryTimewindow(STsdbQueryHandle* pQueryHandle, STsdbQueryCond* pCond) { static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, STsdbQueryCond* pCond) {
pQueryHandle->window = pCond->twindow; pTsdbReadHandle->window = pCond->twindow;
bool updateTs = false; bool updateTs = false;
int64_t startTs = getEarliestValidTimestamp(pQueryHandle->pTsdb); int64_t startTs = getEarliestValidTimestamp(pTsdbReadHandle->pTsdb);
if (ASCENDING_TRAVERSE(pQueryHandle->order)) { if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
if (startTs > pQueryHandle->window.skey) { if (startTs > pTsdbReadHandle->window.skey) {
pQueryHandle->window.skey = startTs; pTsdbReadHandle->window.skey = startTs;
pCond->twindow.skey = startTs; pCond->twindow.skey = startTs;
updateTs = true; updateTs = true;
} }
} else { } else {
if (startTs > pQueryHandle->window.ekey) { if (startTs > pTsdbReadHandle->window.ekey) {
pQueryHandle->window.ekey = startTs; pTsdbReadHandle->window.ekey = startTs;
pCond->twindow.ekey = startTs; pCond->twindow.ekey = startTs;
updateTs = true; updateTs = true;
} }
...@@ -397,51 +398,50 @@ static void setQueryTimewindow(STsdbQueryHandle* pQueryHandle, STsdbQueryCond* p ...@@ -397,51 +398,50 @@ static void setQueryTimewindow(STsdbQueryHandle* pQueryHandle, STsdbQueryCond* p
if (updateTs) { if (updateTs) {
tsdbDebug("%p update the query time window, old:%" PRId64 " - %" PRId64 ", new:%" PRId64 " - %" PRId64 tsdbDebug("%p update the query time window, old:%" PRId64 " - %" PRId64 ", new:%" PRId64 " - %" PRId64
", 0x%" PRIx64, pQueryHandle, pCond->twindow.skey, pCond->twindow.ekey, pQueryHandle->window.skey, ", 0x%" PRIx64, pTsdbReadHandle, pCond->twindow.skey, pCond->twindow.ekey, pTsdbReadHandle->window.skey,
pQueryHandle->window.ekey, pQueryHandle->qId); pTsdbReadHandle->window.ekey, pTsdbReadHandle->qId);
} }
} }
static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pCond, uint64_t qId, SMemRef* pMemRef) { static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, uint64_t qId, STsdbMemTable* pMemRef) {
STsdbQueryHandle* pQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); STsdbReadHandle* pReadHandle = calloc(1, sizeof(STsdbReadHandle));
if (pQueryHandle == NULL) { if (pReadHandle == NULL) {
goto _end; goto _end;
} }
pQueryHandle->order = pCond->order; pReadHandle->order = pCond->order;
pQueryHandle->pTsdb = tsdb; pReadHandle->pTsdb = tsdb;
pQueryHandle->type = TSDB_QUERY_TYPE_ALL; pReadHandle->type = TSDB_QUERY_TYPE_ALL;
pQueryHandle->cur.fid = INT32_MIN; pReadHandle->cur.fid = INT32_MIN;
pQueryHandle->cur.win = TSWINDOW_INITIALIZER; pReadHandle->cur.win = TSWINDOW_INITIALIZER;
pQueryHandle->checkFiles = true; pReadHandle->checkFiles = true;
pQueryHandle->activeIndex = 0; // current active table index pReadHandle->activeIndex = 0; // current active table index
pQueryHandle->qId = qId; pReadHandle->qId = qId;
pQueryHandle->allocSize = 0; pReadHandle->allocSize = 0;
pQueryHandle->locateStart = false; pReadHandle->locateStart = false;
pQueryHandle->pMemRef = pMemRef; pReadHandle->loadType = pCond->type;
pQueryHandle->loadType = pCond->type;
pReadHandle->outputCapacity = 4096;//((STsdb*)tsdb)->config.maxRowsPerFileBlock;
pQueryHandle->outputCapacity = ((STsdbRepo*)tsdb)->config.maxRowsPerFileBlock; pReadHandle->loadExternalRow = pCond->loadExternalRows;
pQueryHandle->loadExternalRow = pCond->loadExternalRows; pReadHandle->currentLoadExternalRows = pCond->loadExternalRows;
pQueryHandle->currentLoadExternalRows = pCond->loadExternalRows;
if (tsdbInitReadH(&pReadHandle->rhelper, (STsdb*)tsdb) != 0) {
if (tsdbInitReadH(&pQueryHandle->rhelper, (STsdbRepo*)tsdb) != 0) {
goto _end; goto _end;
} }
assert(pCond != NULL && pMemRef != NULL); assert(pCond != NULL);
setQueryTimewindow(pQueryHandle, pCond); setQueryTimewindow(pReadHandle, pCond);
if (pCond->numOfCols > 0) { if (pCond->numOfCols > 0) {
// allocate buffer in order to load data blocks from file // allocate buffer in order to load data blocks from file
pQueryHandle->statis = calloc(pCond->numOfCols, sizeof(SDataStatis)); pReadHandle->statis = calloc(pCond->numOfCols, sizeof(SDataStatis));
if (pQueryHandle->statis == NULL) { if (pReadHandle->statis == NULL) {
goto _end; goto _end;
} }
// todo: use list instead of array? // todo: use list instead of array?
pQueryHandle->pColumns = taosArrayInit(pCond->numOfCols, sizeof(SColumnInfoData)); pReadHandle->pColumns = taosArrayInit(pCond->numOfCols, sizeof(SColumnInfoData));
if (pQueryHandle->pColumns == NULL) { if (pReadHandle->pColumns == NULL) {
goto _end; goto _end;
} }
...@@ -449,147 +449,144 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pC ...@@ -449,147 +449,144 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pC
SColumnInfoData colInfo = {{0}, 0}; SColumnInfoData colInfo = {{0}, 0};
colInfo.info = pCond->colList[i]; colInfo.info = pCond->colList[i];
colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCond->colList[i].bytes); colInfo.pData = calloc(1, EXTRA_BYTES + pReadHandle->outputCapacity * pCond->colList[i].bytes);
if (colInfo.pData == NULL) { if (colInfo.pData == NULL) {
goto _end; goto _end;
} }
taosArrayPush(pQueryHandle->pColumns, &colInfo); taosArrayPush(pReadHandle->pColumns, &colInfo);
pQueryHandle->statis[i].colId = colInfo.info.colId; pReadHandle->statis[i].colId = colInfo.info.colId;
} }
pQueryHandle->defaultLoadColumn = getDefaultLoadColumns(pQueryHandle, true); pReadHandle->defaultLoadColumn = getDefaultLoadColumns(pReadHandle, true);
} }
STsdbMeta* pMeta = tsdbGetMeta(tsdb); // STsdbMeta* pMeta = NULL;//tsdbGetMeta(tsdb);
assert(pMeta != NULL); // assert(pMeta != NULL);
pQueryHandle->pDataCols = tdNewDataCols(pMeta->maxCols, pQueryHandle->pTsdb->config.maxRowsPerFileBlock); pReadHandle->pDataCols = tdNewDataCols(1000, pReadHandle->pTsdb->config.maxRowsPerFileBlock);
if (pQueryHandle->pDataCols == NULL) { if (pReadHandle->pDataCols == NULL) {
tsdbError("%p failed to malloc buf for pDataCols, %"PRIu64, pQueryHandle, pQueryHandle->qId); tsdbError("%p failed to malloc buf for pDataCols, %"PRIu64, pReadHandle, pReadHandle->qId);
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
goto _end; goto _end;
} }
tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); tsdbInitDataBlockLoadInfo(&pReadHandle->dataBlockLoadInfo);
tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); tsdbInitCompBlockLoadInfo(&pReadHandle->compBlockLoadInfo);
return (TsdbQueryHandleT) pQueryHandle; return (tsdbReadHandleT)pReadHandle;
_end: _end:
tsdbCleanupQueryHandle(pQueryHandle); // tsdbCleanupQueryHandle(pTsdbReadHandle);
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return NULL; return NULL;
} }
TsdbQueryHandleT* tsdbQueryTables(STsdbRepo* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId, SMemRef* pRef) { tsdbReadHandleT* tsdbQueryTables(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId, void* pRef) {
STsdbQueryHandle* pQueryHandle = tsdbQueryTablesImpl(tsdb, pCond, qId, pRef); STsdbReadHandle* pTsdbReadHandle = tsdbQueryTablesImpl(tsdb, pCond, qId, pRef);
if (pQueryHandle == NULL) { if (pTsdbReadHandle == NULL) {
return NULL; return NULL;
} }
if (emptyQueryTimewindow(pQueryHandle)) { if (emptyQueryTimewindow(pTsdbReadHandle)) {
return (TsdbQueryHandleT*) pQueryHandle; return (tsdbReadHandleT*) pTsdbReadHandle;
} }
STsdbMeta* pMeta = tsdbGetMeta(tsdb);
assert(pMeta != NULL);
SArray* psTable = NULL; SArray* psTable = NULL;
// todo apply the lastkey of table check to avoid to load header file // todo apply the lastkey of table check to avoid to load header file
pQueryHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pQueryHandle, groupList, pMeta, &psTable); pTsdbReadHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pTsdbReadHandle, groupList, &psTable);
if (pQueryHandle->pTableCheckInfo == NULL) { if (pTsdbReadHandle->pTableCheckInfo == NULL) {
tsdbCleanupQueryHandle(pQueryHandle); // tsdbCleanupQueryHandle(pTsdbReadHandle);
taosArrayDestroy(psTable); taosArrayDestroy(psTable);
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return NULL; return NULL;
} }
tsdbMayTakeMemSnapshot(pQueryHandle, psTable); // tsdbMayTakeMemSnapshot(pTsdbReadHandle, psTable);
tsdbDebug("%p total numOfTable:%" PRIzu " in query, 0x%"PRIx64, pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo), pQueryHandle->qId); tsdbDebug("%p total numOfTable:%" PRIzu " in query, 0x%"PRIx64, pTsdbReadHandle, taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo), pTsdbReadHandle->qId);
return (TsdbQueryHandleT) pQueryHandle; return (tsdbReadHandleT) pTsdbReadHandle;
} }
void tsdbResetQueryHandle(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond) { void tsdbResetQueryHandle(tsdbReadHandleT queryHandle, STsdbQueryCond *pCond) {
STsdbQueryHandle* pQueryHandle = queryHandle; STsdbReadHandle* pTsdbReadHandle = queryHandle;
if (emptyQueryTimewindow(pQueryHandle)) { if (emptyQueryTimewindow(pTsdbReadHandle)) {
if (pCond->order != pQueryHandle->order) { if (pCond->order != pTsdbReadHandle->order) {
pQueryHandle->order = pCond->order; pTsdbReadHandle->order = pCond->order;
SWAP(pQueryHandle->window.skey, pQueryHandle->window.ekey, int64_t); SWAP(pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, int64_t);
} }
return; return;
} }
pQueryHandle->order = pCond->order; pTsdbReadHandle->order = pCond->order;
pQueryHandle->window = pCond->twindow; pTsdbReadHandle->window = pCond->twindow;
pQueryHandle->type = TSDB_QUERY_TYPE_ALL; pTsdbReadHandle->type = TSDB_QUERY_TYPE_ALL;
pQueryHandle->cur.fid = -1; pTsdbReadHandle->cur.fid = -1;
pQueryHandle->cur.win = TSWINDOW_INITIALIZER; pTsdbReadHandle->cur.win = TSWINDOW_INITIALIZER;
pQueryHandle->checkFiles = true; pTsdbReadHandle->checkFiles = true;
pQueryHandle->activeIndex = 0; // current active table index pTsdbReadHandle->activeIndex = 0; // current active table index
pQueryHandle->locateStart = false; pTsdbReadHandle->locateStart = false;
pQueryHandle->loadExternalRow = pCond->loadExternalRows; pTsdbReadHandle->loadExternalRow = pCond->loadExternalRows;
if (ASCENDING_TRAVERSE(pCond->order)) { if (ASCENDING_TRAVERSE(pCond->order)) {
assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey); assert(pTsdbReadHandle->window.skey <= pTsdbReadHandle->window.ekey);
} else { } else {
assert(pQueryHandle->window.skey >= pQueryHandle->window.ekey); assert(pTsdbReadHandle->window.skey >= pTsdbReadHandle->window.ekey);
} }
// allocate buffer in order to load data blocks from file // allocate buffer in order to load data blocks from file
memset(pQueryHandle->statis, 0, sizeof(SDataStatis)); memset(pTsdbReadHandle->statis, 0, sizeof(SDataStatis));
tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); tsdbInitDataBlockLoadInfo(&pTsdbReadHandle->dataBlockLoadInfo);
tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); tsdbInitCompBlockLoadInfo(&pTsdbReadHandle->compBlockLoadInfo);
resetCheckInfo(pQueryHandle); resetCheckInfo(pTsdbReadHandle);
} }
void tsdbResetQueryHandleForNewTable(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond, STableGroupInfo* groupList) { void tsdbResetQueryHandleForNewTable(tsdbReadHandleT queryHandle, STsdbQueryCond *pCond, STableGroupInfo* groupList) {
STsdbQueryHandle* pQueryHandle = queryHandle; STsdbReadHandle* pTsdbReadHandle = queryHandle;
pQueryHandle->order = pCond->order; pTsdbReadHandle->order = pCond->order;
pQueryHandle->window = pCond->twindow; pTsdbReadHandle->window = pCond->twindow;
pQueryHandle->type = TSDB_QUERY_TYPE_ALL; pTsdbReadHandle->type = TSDB_QUERY_TYPE_ALL;
pQueryHandle->cur.fid = -1; pTsdbReadHandle->cur.fid = -1;
pQueryHandle->cur.win = TSWINDOW_INITIALIZER; pTsdbReadHandle->cur.win = TSWINDOW_INITIALIZER;
pQueryHandle->checkFiles = true; pTsdbReadHandle->checkFiles = true;
pQueryHandle->activeIndex = 0; // current active table index pTsdbReadHandle->activeIndex = 0; // current active table index
pQueryHandle->locateStart = false; pTsdbReadHandle->locateStart = false;
pQueryHandle->loadExternalRow = pCond->loadExternalRows; pTsdbReadHandle->loadExternalRow = pCond->loadExternalRows;
if (ASCENDING_TRAVERSE(pCond->order)) { if (ASCENDING_TRAVERSE(pCond->order)) {
assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey); assert(pTsdbReadHandle->window.skey <= pTsdbReadHandle->window.ekey);
} else { } else {
assert(pQueryHandle->window.skey >= pQueryHandle->window.ekey); assert(pTsdbReadHandle->window.skey >= pTsdbReadHandle->window.ekey);
} }
// allocate buffer in order to load data blocks from file // allocate buffer in order to load data blocks from file
memset(pQueryHandle->statis, 0, sizeof(SDataStatis)); memset(pTsdbReadHandle->statis, 0, sizeof(SDataStatis));
tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); tsdbInitDataBlockLoadInfo(&pTsdbReadHandle->dataBlockLoadInfo);
tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); tsdbInitCompBlockLoadInfo(&pTsdbReadHandle->compBlockLoadInfo);
SArray* pTable = NULL; SArray* pTable = NULL;
STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb); // STsdbMeta* pMeta = tsdbGetMeta(pTsdbReadHandle->pTsdb);
pQueryHandle->pTableCheckInfo = destroyTableCheckInfo(pQueryHandle->pTableCheckInfo); // pTsdbReadHandle->pTableCheckInfo = destroyTableCheckInfo(pTsdbReadHandle->pTableCheckInfo);
pQueryHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pQueryHandle, groupList, pMeta, &pTable); pTsdbReadHandle->pTableCheckInfo = NULL;//createCheckInfoFromTableGroup(pTsdbReadHandle, groupList, pMeta, &pTable);
if (pQueryHandle->pTableCheckInfo == NULL) { if (pTsdbReadHandle->pTableCheckInfo == NULL) {
tsdbCleanupQueryHandle(pQueryHandle); // tsdbCleanupQueryHandle(pTsdbReadHandle);
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
} }
pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); // pTsdbReadHandle->prev = doFreeColumnInfoData(pTsdbReadHandle->prev);
pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); // pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next);
} }
TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pMemRef) { tsdbReadHandleT tsdbQueryLastRow(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) {
pCond->twindow = updateLastrowForEachGroup(groupList); pCond->twindow = updateLastrowForEachGroup(groupList);
// no qualified table // no qualified table
...@@ -597,56 +594,56 @@ TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STable ...@@ -597,56 +594,56 @@ TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STable
return NULL; return NULL;
} }
STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef);
if (pQueryHandle == NULL) { if (pTsdbReadHandle == NULL) {
return NULL; return NULL;
} }
int32_t code = checkForCachedLastRow(pQueryHandle, groupList); int32_t code = checkForCachedLastRow(pTsdbReadHandle, groupList);
if (code != TSDB_CODE_SUCCESS) { // set the numOfTables to be 0 if (code != TSDB_CODE_SUCCESS) { // set the numOfTables to be 0
terrno = code; terrno = code;
return NULL; return NULL;
} }
assert(pCond->order == TSDB_ORDER_ASC && pCond->twindow.skey <= pCond->twindow.ekey); assert(pCond->order == TSDB_ORDER_ASC && pCond->twindow.skey <= pCond->twindow.ekey);
if (pQueryHandle->cachelastrow) { if (pTsdbReadHandle->cachelastrow) {
pQueryHandle->type = TSDB_QUERY_TYPE_LAST; pTsdbReadHandle->type = TSDB_QUERY_TYPE_LAST;
} }
return pQueryHandle; return pTsdbReadHandle;
} }
#if 0
TsdbQueryHandleT tsdbQueryCacheLast(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pMemRef) { tsdbReadHandleT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) {
STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef);
if (pQueryHandle == NULL) { if (pTsdbReadHandle == NULL) {
return NULL; return NULL;
} }
int32_t code = checkForCachedLast(pQueryHandle); int32_t code = checkForCachedLast(pTsdbReadHandle);
if (code != TSDB_CODE_SUCCESS) { // set the numOfTables to be 0 if (code != TSDB_CODE_SUCCESS) { // set the numOfTables to be 0
terrno = code; terrno = code;
return NULL; return NULL;
} }
if (pQueryHandle->cachelastrow) { if (pTsdbReadHandle->cachelastrow) {
pQueryHandle->type = TSDB_QUERY_TYPE_LAST; pTsdbReadHandle->type = TSDB_QUERY_TYPE_LAST;
} }
return pQueryHandle; return pTsdbReadHandle;
} }
#endif
SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle) { SArray* tsdbGetQueriedTableList(tsdbReadHandleT *pHandle) {
assert(pHandle != NULL); assert(pHandle != NULL);
STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) pHandle; STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) pHandle;
size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); size_t size = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo);
SArray* res = taosArrayInit(size, POINTER_BYTES); SArray* res = taosArrayInit(size, POINTER_BYTES);
for(int32_t i = 0; i < size; ++i) { for(int32_t i = 0; i < size; ++i) {
STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i);
taosArrayPush(res, &pCheckInfo->pTableObj); taosArrayPush(res, &pCheckInfo->pTableObj);
} }
...@@ -668,11 +665,11 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr ...@@ -668,11 +665,11 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr
SArray* px = taosArrayInit(4, sizeof(STableKeyInfo)); SArray* px = taosArrayInit(4, sizeof(STableKeyInfo));
for (int32_t j = 0; j < numOfTables; ++j) { for (int32_t j = 0; j < numOfTables; ++j) {
STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(oneGroup, j); STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(oneGroup, j);
if (window->skey <= pInfo->lastKey && ((STable*)pInfo->pTable)->lastKey != TSKEY_INITIAL_VAL) { // if (window->skey <= pInfo->lastKey && ((STable*)pInfo->pTable)->lastKey != TSKEY_INITIAL_VAL) {
taosArrayPush(px, pInfo); // taosArrayPush(px, pInfo);
pNew->numOfTables += 1; // pNew->numOfTables += 1;
break; // break;
} // }
} }
// there are no data in this group // there are no data in this group
...@@ -686,7 +683,7 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr ...@@ -686,7 +683,7 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr
return pNew; return pNew;
} }
TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pRef) { tsdbReadHandleT tsdbQueryRowsInExternalWindow(STsdb *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pRef) {
STableGroupInfo* pNew = trimTableGroup(&pCond->twindow, groupList); STableGroupInfo* pNew = trimTableGroup(&pCond->twindow, groupList);
if (pNew->numOfTables == 0) { if (pNew->numOfTables == 0) {
...@@ -701,17 +698,14 @@ TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond* ...@@ -701,17 +698,14 @@ TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond*
} }
} }
STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, pNew, qId, pRef); STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, pNew, qId, pRef);
pQueryHandle->loadExternalRow = true; pTsdbReadHandle->loadExternalRow = true;
pQueryHandle->currentLoadExternalRows = true; pTsdbReadHandle->currentLoadExternalRows = true;
return pQueryHandle; return pTsdbReadHandle;
} }
static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCheckInfo) { static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pCheckInfo) {
STable* pTable = pCheckInfo->pTableObj;
assert(pTable != NULL);
if (pCheckInfo->initBuf) { if (pCheckInfo->initBuf) {
return true; return true;
} }
...@@ -720,33 +714,29 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh ...@@ -720,33 +714,29 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
int32_t order = pHandle->order; int32_t order = pHandle->order;
// no data in buffer, abort // no data in buffer, abort
if (pHandle->pMemRef->snapshot.mem == NULL && pHandle->pMemRef->snapshot.imem == NULL) { // if (pHandle->pMemTable->snapshot.mem == NULL && pHandle->pMemTable->snapshot.imem == NULL) {
return false; // return false;
} // }
//
assert(pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL); // assert(pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL);
//
STableData* pMem = NULL; STbData** pMem = NULL;
STableData* pIMem = NULL; STbData** pIMem = NULL;
SMemTable* pMemT = pHandle->pMemRef->snapshot.mem; TKEY tLastKey = 0; /// keyToTkey(pCheckInfo->lastKey);
SMemTable* pIMemT = pHandle->pMemRef->snapshot.imem; if (pHandle->pTsdb->mem != NULL) {
pMem = taosHashGet(pHandle->pTsdb->mem->pHashIdx, &pCheckInfo->tableId, sizeof(pCheckInfo->tableId));
if (pMemT && pCheckInfo->tableId.tid < pMemT->maxTables) { if (pMem != NULL) {
pMem = pMemT->tData[pCheckInfo->tableId.tid];
if (pMem != NULL && pMem->uid == pCheckInfo->tableId.uid) { // check uid
TKEY tLastKey = keyToTkey(pCheckInfo->lastKey);
pCheckInfo->iter = pCheckInfo->iter =
tSkipListCreateIterFromVal(pMem->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order); tSkipListCreateIterFromVal((*pMem)->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order);
} }
} }
if (pIMemT && pCheckInfo->tableId.tid < pIMemT->maxTables) { if (pHandle->pTsdb->imem != NULL) {
pIMem = pIMemT->tData[pCheckInfo->tableId.tid]; pIMem = taosHashGet(pHandle->pTsdb->imem->pHashIdx, &pCheckInfo->tableId, sizeof(pCheckInfo->tableId));
if (pIMem != NULL && pIMem->uid == pCheckInfo->tableId.uid) { // check uid if (pIMem != NULL) {
TKEY tLastKey = keyToTkey(pCheckInfo->lastKey);
pCheckInfo->iiter = pCheckInfo->iiter =
tSkipListCreateIterFromVal(pIMem->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order); tSkipListCreateIterFromVal((*pIMem)->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order);
} }
} }
...@@ -767,10 +757,9 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh ...@@ -767,10 +757,9 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
SMemRow row = (SMemRow)SL_GET_NODE_DATA(node); SMemRow row = (SMemRow)SL_GET_NODE_DATA(node);
TSKEY key = memRowKey(row); // first timestamp in buffer TSKEY key = memRowKey(row); // first timestamp in buffer
tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 tsdbDebug("%p uid:%" PRId64 ", check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
"-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", 0x%"PRIx64, "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", 0x%"PRIx64,
pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pMem->keyFirst, pMem->keyLast, pHandle, pCheckInfo->tableId, key, order, (*pMem)->keyMin, (*pMem)->keyMax, pCheckInfo->lastKey, (*pMem)->nrows, pHandle->qId);
pCheckInfo->lastKey, pMem->numOfRows, pHandle->qId);
if (ASCENDING_TRAVERSE(order)) { if (ASCENDING_TRAVERSE(order)) {
assert(pCheckInfo->lastKey <= key); assert(pCheckInfo->lastKey <= key);
...@@ -779,8 +768,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh ...@@ -779,8 +768,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
} }
} else { } else {
tsdbDebug("%p uid:%"PRId64", tid:%d no data in mem, 0x%"PRIx64, pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, tsdbDebug("%p uid:%"PRId64", no data in mem, 0x%"PRIx64, pHandle, pCheckInfo->tableId, pHandle->qId);
pHandle->qId);
} }
if (!imemEmpty) { if (!imemEmpty) {
...@@ -789,10 +777,9 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh ...@@ -789,10 +777,9 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
SMemRow row = (SMemRow)SL_GET_NODE_DATA(node); SMemRow row = (SMemRow)SL_GET_NODE_DATA(node);
TSKEY key = memRowKey(row); // first timestamp in buffer TSKEY key = memRowKey(row); // first timestamp in buffer
tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 tsdbDebug("%p uid:%" PRId64 ", check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
"-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", 0x%"PRIx64, "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", 0x%"PRIx64,
pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pIMem->keyFirst, pIMem->keyLast, pHandle, pCheckInfo->tableId, key, order, (*pIMem)->keyMin, (*pIMem)->keyMax, pCheckInfo->lastKey, (*pIMem)->nrows, pHandle->qId);
pCheckInfo->lastKey, pIMem->numOfRows, pHandle->qId);
if (ASCENDING_TRAVERSE(order)) { if (ASCENDING_TRAVERSE(order)) {
assert(pCheckInfo->lastKey <= key); assert(pCheckInfo->lastKey <= key);
...@@ -800,7 +787,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh ...@@ -800,7 +787,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
assert(pCheckInfo->lastKey >= key); assert(pCheckInfo->lastKey >= key);
} }
} else { } else {
tsdbDebug("%p uid:%"PRId64", tid:%d no data in imem, 0x%"PRIx64, pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, tsdbDebug("%p uid:%"PRId64", no data in imem, 0x%"PRIx64, pHandle, pCheckInfo->tableId,
pHandle->qId); pHandle->qId);
} }
...@@ -973,17 +960,13 @@ static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) { ...@@ -973,17 +960,13 @@ static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) {
return hasNext; return hasNext;
} }
static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { static bool hasMoreDataInCache(STsdbReadHandle* pHandle) {
STsdbCfg *pCfg = &pHandle->pTsdb->config; STsdbCfg *pCfg = &pHandle->pTsdb->config;
size_t size = taosArrayGetSize(pHandle->pTableCheckInfo); size_t size = taosArrayGetSize(pHandle->pTableCheckInfo);
assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1); assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1);
pHandle->cur.fid = INT32_MIN; pHandle->cur.fid = INT32_MIN;
STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex);
STable* pTable = pCheckInfo->pTableObj;
assert(pTable != NULL);
if (!pCheckInfo->initBuf) { if (!pCheckInfo->initBuf) {
initTableMemIterator(pHandle, pCheckInfo); initTableMemIterator(pHandle, pCheckInfo);
} }
...@@ -994,8 +977,8 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { ...@@ -994,8 +977,8 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
} }
pCheckInfo->lastKey = memRowKey(row); // first timestamp in buffer pCheckInfo->lastKey = memRowKey(row); // first timestamp in buffer
tsdbDebug("%p uid:%" PRId64", tid:%d check data in buffer from skey:%" PRId64 ", order:%d, 0x%"PRIx64, pHandle, tsdbDebug("%p uid:%" PRId64", check data in buffer from skey:%" PRId64 ", order:%d, 0x%"PRIx64, pHandle,
pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pCheckInfo->lastKey, pHandle->order, pHandle->qId); pCheckInfo->tableId, pCheckInfo->lastKey, pHandle->order, pHandle->qId);
// all data in mem are checked already. // all data in mem are checked already.
if ((pCheckInfo->lastKey > pHandle->window.ekey && ASCENDING_TRAVERSE(pHandle->order)) || if ((pCheckInfo->lastKey > pHandle->window.ekey && ASCENDING_TRAVERSE(pHandle->order)) ||
...@@ -1068,21 +1051,21 @@ static int32_t binarySearchForBlock(SBlock* pBlock, int32_t numOfBlocks, TSKEY s ...@@ -1068,21 +1051,21 @@ static int32_t binarySearchForBlock(SBlock* pBlock, int32_t numOfBlocks, TSKEY s
return midSlot; return midSlot;
} }
static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t index, int32_t* numOfBlocks) { static int32_t loadBlockInfo(STsdbReadHandle * pTsdbReadHandle, int32_t index, int32_t* numOfBlocks) {
int32_t code = 0; int32_t code = 0;
STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, index); STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, index);
pCheckInfo->numOfBlocks = 0; pCheckInfo->numOfBlocks = 0;
if (tsdbSetReadTable(&pQueryHandle->rhelper, pCheckInfo->pTableObj) != TSDB_CODE_SUCCESS) { if (tsdbSetReadTable(&pTsdbReadHandle->rhelper, pCheckInfo->pTableObj) != TSDB_CODE_SUCCESS) {
code = terrno; code = terrno;
return code; return code;
} }
SBlockIdx* compIndex = pQueryHandle->rhelper.pBlkIdx; SBlockIdx* compIndex = pTsdbReadHandle->rhelper.pBlkIdx;
// no data block in this file, try next file // no data block in this file, try next file
if (compIndex == NULL || compIndex->uid != pCheckInfo->tableId.uid) { if (compIndex == NULL || compIndex->uid != pCheckInfo->tableId) {
return 0; // no data blocks in the file belongs to pCheckInfo->pTable return 0; // no data blocks in the file belongs to pCheckInfo->pTable
} }
...@@ -1100,21 +1083,21 @@ static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t index, int ...@@ -1100,21 +1083,21 @@ static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t index, int
pCheckInfo->compSize = compIndex->len; pCheckInfo->compSize = compIndex->len;
} }
if (tsdbLoadBlockInfo(&(pQueryHandle->rhelper), (void*)(pCheckInfo->pCompInfo)) < 0) { if (tsdbLoadBlockInfo(&(pTsdbReadHandle->rhelper), (void*)(pCheckInfo->pCompInfo)) < 0) {
return terrno; return terrno;
} }
SBlockInfo* pCompInfo = pCheckInfo->pCompInfo; SBlockInfo* pCompInfo = pCheckInfo->pCompInfo;
TSKEY s = TSKEY_INITIAL_VAL, e = TSKEY_INITIAL_VAL; TSKEY s = TSKEY_INITIAL_VAL, e = TSKEY_INITIAL_VAL;
if (ASCENDING_TRAVERSE(pQueryHandle->order)) { if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
assert(pCheckInfo->lastKey <= pQueryHandle->window.ekey && pQueryHandle->window.skey <= pQueryHandle->window.ekey); assert(pCheckInfo->lastKey <= pTsdbReadHandle->window.ekey && pTsdbReadHandle->window.skey <= pTsdbReadHandle->window.ekey);
} else { } else {
assert(pCheckInfo->lastKey >= pQueryHandle->window.ekey && pQueryHandle->window.skey >= pQueryHandle->window.ekey); assert(pCheckInfo->lastKey >= pTsdbReadHandle->window.ekey && pTsdbReadHandle->window.skey >= pTsdbReadHandle->window.ekey);
} }
s = MIN(pCheckInfo->lastKey, pQueryHandle->window.ekey); s = MIN(pCheckInfo->lastKey, pTsdbReadHandle->window.ekey);
e = MAX(pCheckInfo->lastKey, pQueryHandle->window.ekey); e = MAX(pCheckInfo->lastKey, pTsdbReadHandle->window.ekey);
// discard the unqualified data block based on the query time window // discard the unqualified data block based on the query time window
int32_t start = binarySearchForBlock(pCompInfo->blocks, compIndex->numOfBlocks, s, TSDB_ORDER_ASC); int32_t start = binarySearchForBlock(pCompInfo->blocks, compIndex->numOfBlocks, s, TSDB_ORDER_ASC);
...@@ -1139,26 +1122,26 @@ static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t index, int ...@@ -1139,26 +1122,26 @@ static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t index, int
return 0; return 0;
} }
static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlocks) { static int32_t getFileCompInfo(STsdbReadHandle* pTsdbReadHandle, int32_t* numOfBlocks) {
// load all the comp offset value for all tables in this file // load all the comp offset value for all tables in this file
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
*numOfBlocks = 0; *numOfBlocks = 0;
pQueryHandle->cost.headFileLoad += 1; pTsdbReadHandle->cost.headFileLoad += 1;
int64_t s = taosGetTimestampUs(); int64_t s = taosGetTimestampUs();
size_t numOfTables = 0; size_t numOfTables = 0;
if (pQueryHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { if (pTsdbReadHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) {
code = loadBlockInfo(pQueryHandle, pQueryHandle->activeIndex, numOfBlocks); code = loadBlockInfo(pTsdbReadHandle, pTsdbReadHandle->activeIndex, numOfBlocks);
} else if (pQueryHandle->loadType == BLOCK_LOAD_OFFSET_SEQ_ORDER) { } else if (pTsdbReadHandle->loadType == BLOCK_LOAD_OFFSET_SEQ_ORDER) {
numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo);
for (int32_t i = 0; i < numOfTables; ++i) { for (int32_t i = 0; i < numOfTables; ++i) {
code = loadBlockInfo(pQueryHandle, i, numOfBlocks); code = loadBlockInfo(pTsdbReadHandle, i, numOfBlocks);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
int64_t e = taosGetTimestampUs(); int64_t e = taosGetTimestampUs();
pQueryHandle->cost.headFileLoadTime += (e - s); pTsdbReadHandle->cost.headFileLoadTime += (e - s);
return code; return code;
} }
} }
...@@ -1167,57 +1150,57 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo ...@@ -1167,57 +1150,57 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
} }
int64_t e = taosGetTimestampUs(); int64_t e = taosGetTimestampUs();
pQueryHandle->cost.headFileLoadTime += (e - s); pTsdbReadHandle->cost.headFileLoadTime += (e - s);
return code; return code;
} }
static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) { static int32_t doLoadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) {
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
STSchema *pSchema = tsdbGetTableSchema(pCheckInfo->pTableObj); STSchema *pSchema = NULL;//tsdbGetTableSchema(pCheckInfo->pTableObj);
int32_t code = tdInitDataCols(pQueryHandle->pDataCols, pSchema); int32_t code = tdInitDataCols(pTsdbReadHandle->pDataCols, pSchema);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tsdbError("%p failed to malloc buf for pDataCols, 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); tsdbError("%p failed to malloc buf for pDataCols, 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId);
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
goto _error; goto _error;
} }
code = tdInitDataCols(pQueryHandle->rhelper.pDCols[0], pSchema); code = tdInitDataCols(pTsdbReadHandle->rhelper.pDCols[0], pSchema);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tsdbError("%p failed to malloc buf for rhelper.pDataCols[0], 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); tsdbError("%p failed to malloc buf for rhelper.pDataCols[0], 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId);
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
goto _error; goto _error;
} }
code = tdInitDataCols(pQueryHandle->rhelper.pDCols[1], pSchema); code = tdInitDataCols(pTsdbReadHandle->rhelper.pDCols[1], pSchema);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tsdbError("%p failed to malloc buf for rhelper.pDataCols[1], 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); tsdbError("%p failed to malloc buf for rhelper.pDataCols[1], 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId);
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
goto _error; goto _error;
} }
int16_t* colIds = pQueryHandle->defaultLoadColumn->pData; int16_t* colIds = pTsdbReadHandle->defaultLoadColumn->pData;
int32_t ret = tsdbLoadBlockDataCols(&(pQueryHandle->rhelper), pBlock, pCheckInfo->pCompInfo, colIds, (int)(QH_GET_NUM_OF_COLS(pQueryHandle))); int32_t ret = tsdbLoadBlockDataCols(&(pTsdbReadHandle->rhelper), pBlock, pCheckInfo->pCompInfo, colIds, (int)(QH_GET_NUM_OF_COLS(pTsdbReadHandle)));
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
int32_t c = terrno; int32_t c = terrno;
assert(c != TSDB_CODE_SUCCESS); assert(c != TSDB_CODE_SUCCESS);
goto _error; goto _error;
} }
SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo; SDataBlockLoadInfo* pBlockLoadInfo = &pTsdbReadHandle->dataBlockLoadInfo;
pBlockLoadInfo->fileGroup = pQueryHandle->pFileGroup; pBlockLoadInfo->fileGroup = pTsdbReadHandle->pFileGroup;
pBlockLoadInfo->slot = pQueryHandle->cur.slot; pBlockLoadInfo->slot = pTsdbReadHandle->cur.slot;
pBlockLoadInfo->tid = pCheckInfo->pTableObj->tableId.tid; pBlockLoadInfo->uid = pCheckInfo->pTableObj->uid;
SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0];
assert(pCols->numOfRows != 0 && pCols->numOfRows <= pBlock->numOfRows); assert(pCols->numOfRows != 0 && pCols->numOfRows <= pBlock->numOfRows);
pBlock->numOfRows = pCols->numOfRows; pBlock->numOfRows = pCols->numOfRows;
// Convert from TKEY to TSKEY for primary timestamp column if current block has timestamp before 1970-01-01T00:00:00Z // Convert from TKEY to TSKEY for primary timestamp column if current block has timestamp before 1970-01-01T00:00:00Z
if(pBlock->keyFirst < 0 && colIds[0] == PRIMARYKEY_TIMESTAMP_COL_INDEX) { if(pBlock->keyFirst < 0 && colIds[0] == PRIMARYKEY_TIMESTAMP_COL_ID) {
int64_t* src = pCols->cols[0].pData; int64_t* src = pCols->cols[0].pData;
for(int32_t i = 0; i < pBlock->numOfRows; ++i) { for(int32_t i = 0; i < pBlock->numOfRows; ++i) {
src[i] = tdGetKey(src[i]); src[i] = tdGetKey(src[i]);
...@@ -1225,60 +1208,60 @@ static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBloc ...@@ -1225,60 +1208,60 @@ static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBloc
} }
int64_t elapsedTime = (taosGetTimestampUs() - st); int64_t elapsedTime = (taosGetTimestampUs() - st);
pQueryHandle->cost.blockLoadTime += elapsedTime; pTsdbReadHandle->cost.blockLoadTime += elapsedTime;
tsdbDebug("%p load file block into buffer, index:%d, brange:%"PRId64"-%"PRId64", rows:%d, elapsed time:%"PRId64 " us, 0x%"PRIx64, tsdbDebug("%p load file block into buffer, index:%d, brange:%"PRId64"-%"PRId64", rows:%d, elapsed time:%"PRId64 " us, 0x%"PRIx64,
pQueryHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, elapsedTime, pQueryHandle->qId); pTsdbReadHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, elapsedTime, pTsdbReadHandle->qId);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
_error: _error:
pBlock->numOfRows = 0; pBlock->numOfRows = 0;
tsdbError("%p error occurs in loading file block, index:%d, brange:%"PRId64"-%"PRId64", rows:%d, 0x%"PRIx64, tsdbError("%p error occurs in loading file block, index:%d, brange:%"PRId64"-%"PRId64", rows:%d, 0x%"PRIx64,
pQueryHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, pQueryHandle->qId); pTsdbReadHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, pTsdbReadHandle->qId);
return terrno; return terrno;
} }
static int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBlockInfo); static int32_t getEndPosInDataBlock(STsdbReadHandle* pTsdbReadHandle, SDataBlockInfo* pBlockInfo);
static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end); static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end);
static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, int32_t numOfCols); static void moveDataToFront(STsdbReadHandle* pTsdbReadHandle, int32_t numOfRows, int32_t numOfCols);
static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle); static void doCheckGeneratedBlockRange(STsdbReadHandle* pTsdbReadHandle);
static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos); static void copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos);
static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo){ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo){
SQueryFilePos* cur = &pQueryHandle->cur; SQueryFilePos* cur = &pTsdbReadHandle->cur;
STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config;
SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock);
TSKEY key; TSKEY key;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
/*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo); /*bool hasData = */ initTableMemIterator(pTsdbReadHandle, pCheckInfo);
assert(cur->pos >= 0 && cur->pos <= binfo.rows); assert(cur->pos >= 0 && cur->pos <= binfo.rows);
key = extractFirstTraverseKey(pCheckInfo, pQueryHandle->order, pCfg->update); key = extractFirstTraverseKey(pCheckInfo, pTsdbReadHandle->order, pCfg->update);
if (key != TSKEY_INITIAL_VAL) { if (key != TSKEY_INITIAL_VAL) {
tsdbDebug("%p key in mem:%"PRId64", 0x%"PRIx64, pQueryHandle, key, pQueryHandle->qId); tsdbDebug("%p key in mem:%"PRId64", 0x%"PRIx64, pTsdbReadHandle, key, pTsdbReadHandle->qId);
} else { } else {
tsdbDebug("%p no data in mem, 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); tsdbDebug("%p no data in mem, 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId);
} }
if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) || if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) ||
(!ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) { (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) {
if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) || if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) ||
(!ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) { (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) {
// do not load file block into buffer // do not load file block into buffer
int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1; int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1;
TSKEY maxKey = ASCENDING_TRAVERSE(pQueryHandle->order)? (binfo.window.skey - step):(binfo.window.ekey - step); TSKEY maxKey = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? (binfo.window.skey - step):(binfo.window.ekey - step);
cur->rows = tsdbReadRowsFromCache(pCheckInfo, maxKey, pQueryHandle->outputCapacity, &cur->win, pQueryHandle); cur->rows = tsdbReadRowsFromCache(pCheckInfo, maxKey, pTsdbReadHandle->outputCapacity, &cur->win, pTsdbReadHandle);
pQueryHandle->realNumOfRows = cur->rows; pTsdbReadHandle->realNumOfRows = cur->rows;
// update the last key value // update the last key value
pCheckInfo->lastKey = cur->win.ekey + step; pCheckInfo->lastKey = cur->win.ekey + step;
if (!ASCENDING_TRAVERSE(pQueryHandle->order)) { if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
SWAP(cur->win.skey, cur->win.ekey, TSKEY); SWAP(cur->win.skey, cur->win.ekey, TSKEY);
} }
...@@ -1289,11 +1272,11 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p ...@@ -1289,11 +1272,11 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p
// return error, add test cases // return error, add test cases
if ((code = doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { if ((code = doLoadFileDataBlock(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) {
return code; return code;
} }
doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); doMergeTwoLevelData(pTsdbReadHandle, pCheckInfo, pBlock);
} else { } else {
/* /*
* no data in cache, only load data from file * no data in cache, only load data from file
...@@ -1301,19 +1284,19 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p ...@@ -1301,19 +1284,19 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p
* *
* Here the buffer is not enough, so only part of file block can be loaded into memory buffer * Here the buffer is not enough, so only part of file block can be loaded into memory buffer
*/ */
assert(pQueryHandle->outputCapacity >= binfo.rows); assert(pTsdbReadHandle->outputCapacity >= binfo.rows);
int32_t endPos = getEndPosInDataBlock(pQueryHandle, &binfo); int32_t endPos = getEndPosInDataBlock(pTsdbReadHandle, &binfo);
if ((cur->pos == 0 && endPos == binfo.rows -1 && ASCENDING_TRAVERSE(pQueryHandle->order)) || if ((cur->pos == 0 && endPos == binfo.rows -1 && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) ||
(cur->pos == (binfo.rows - 1) && endPos == 0 && (!ASCENDING_TRAVERSE(pQueryHandle->order)))) { (cur->pos == (binfo.rows - 1) && endPos == 0 && (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)))) {
pQueryHandle->realNumOfRows = binfo.rows; pTsdbReadHandle->realNumOfRows = binfo.rows;
cur->rows = binfo.rows; cur->rows = binfo.rows;
cur->win = binfo.window; cur->win = binfo.window;
cur->mixBlock = false; cur->mixBlock = false;
cur->blockCompleted = true; cur->blockCompleted = true;
if (ASCENDING_TRAVERSE(pQueryHandle->order)) { if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
cur->lastKey = binfo.window.ekey + 1; cur->lastKey = binfo.window.ekey + 1;
cur->pos = binfo.rows; cur->pos = binfo.rows;
} else { } else {
...@@ -1321,17 +1304,17 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p ...@@ -1321,17 +1304,17 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p
cur->pos = -1; cur->pos = -1;
} }
} else { // partially copy to dest buffer } else { // partially copy to dest buffer
copyAllRemainRowsFromFileBlock(pQueryHandle, pCheckInfo, &binfo, endPos); copyAllRemainRowsFromFileBlock(pTsdbReadHandle, pCheckInfo, &binfo, endPos);
cur->mixBlock = true; cur->mixBlock = true;
} }
assert(cur->blockCompleted); assert(cur->blockCompleted);
if (cur->rows == binfo.rows) { if (cur->rows == binfo.rows) {
tsdbDebug("%p whole file block qualified, brange:%"PRId64"-%"PRId64", rows:%d, lastKey:%"PRId64", %"PRIx64, tsdbDebug("%p whole file block qualified, brange:%"PRId64"-%"PRId64", rows:%d, lastKey:%"PRId64", %"PRIx64,
pQueryHandle, cur->win.skey, cur->win.ekey, cur->rows, cur->lastKey, pQueryHandle->qId); pTsdbReadHandle, cur->win.skey, cur->win.ekey, cur->rows, cur->lastKey, pTsdbReadHandle->qId);
} else { } else {
tsdbDebug("%p create data block from remain file block, brange:%"PRId64"-%"PRId64", rows:%d, total:%d, lastKey:%"PRId64", %"PRIx64, tsdbDebug("%p create data block from remain file block, brange:%"PRId64"-%"PRId64", rows:%d, total:%d, lastKey:%"PRId64", %"PRIx64,
pQueryHandle, cur->win.skey, cur->win.ekey, cur->rows, binfo.rows, cur->lastKey, pQueryHandle->qId); pTsdbReadHandle, cur->win.skey, cur->win.ekey, cur->rows, binfo.rows, cur->lastKey, pTsdbReadHandle->qId);
} }
} }
...@@ -1339,58 +1322,58 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p ...@@ -1339,58 +1322,58 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p
return code; return code;
} }
static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, bool* exists) { static int32_t loadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, bool* exists) {
SQueryFilePos* cur = &pQueryHandle->cur; SQueryFilePos* cur = &pTsdbReadHandle->cur;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
bool asc = ASCENDING_TRAVERSE(pQueryHandle->order); bool asc = ASCENDING_TRAVERSE(pTsdbReadHandle->order);
if (asc) { if (asc) {
// query ended in/started from current block // query ended in/started from current block
if (pQueryHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) { if (pTsdbReadHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) {
if ((code = doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { if ((code = doLoadFileDataBlock(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) {
*exists = false; *exists = false;
return code; return code;
} }
SDataCols* pTSCol = pQueryHandle->rhelper.pDCols[0]; SDataCols* pTSCol = pTsdbReadHandle->rhelper.pDCols[0];
assert(pTSCol->cols->type == TSDB_DATA_TYPE_TIMESTAMP && pTSCol->numOfRows == pBlock->numOfRows); assert(pTSCol->cols->type == TSDB_DATA_TYPE_TIMESTAMP && pTSCol->numOfRows == pBlock->numOfRows);
if (pCheckInfo->lastKey > pBlock->keyFirst) { if (pCheckInfo->lastKey > pBlock->keyFirst) {
cur->pos = cur->pos =
binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order); binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pTsdbReadHandle->order);
} else { } else {
cur->pos = 0; cur->pos = 0;
} }
assert(pCheckInfo->lastKey <= pBlock->keyLast); assert(pCheckInfo->lastKey <= pBlock->keyLast);
doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); doMergeTwoLevelData(pTsdbReadHandle, pCheckInfo, pBlock);
} else { // the whole block is loaded in to buffer } else { // the whole block is loaded in to buffer
cur->pos = asc? 0:(pBlock->numOfRows - 1); cur->pos = asc? 0:(pBlock->numOfRows - 1);
code = handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo); code = handleDataMergeIfNeeded(pTsdbReadHandle, pBlock, pCheckInfo);
} }
} else { //desc order, query ended in current block } else { //desc order, query ended in current block
if (pQueryHandle->window.ekey > pBlock->keyFirst || pCheckInfo->lastKey < pBlock->keyLast) { if (pTsdbReadHandle->window.ekey > pBlock->keyFirst || pCheckInfo->lastKey < pBlock->keyLast) {
if ((code = doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { if ((code = doLoadFileDataBlock(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) {
*exists = false; *exists = false;
return code; return code;
} }
SDataCols* pTsCol = pQueryHandle->rhelper.pDCols[0]; SDataCols* pTsCol = pTsdbReadHandle->rhelper.pDCols[0];
if (pCheckInfo->lastKey < pBlock->keyLast) { if (pCheckInfo->lastKey < pBlock->keyLast) {
cur->pos = binarySearchForKey(pTsCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order); cur->pos = binarySearchForKey(pTsCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pTsdbReadHandle->order);
} else { } else {
cur->pos = pBlock->numOfRows - 1; cur->pos = pBlock->numOfRows - 1;
} }
assert(pCheckInfo->lastKey >= pBlock->keyFirst); assert(pCheckInfo->lastKey >= pBlock->keyFirst);
doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); doMergeTwoLevelData(pTsdbReadHandle, pCheckInfo, pBlock);
} else { } else {
cur->pos = asc? 0:(pBlock->numOfRows-1); cur->pos = asc? 0:(pBlock->numOfRows-1);
code = handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo); code = handleDataMergeIfNeeded(pTsdbReadHandle, pBlock, pCheckInfo);
} }
} }
*exists = pQueryHandle->realNumOfRows > 0; *exists = pTsdbReadHandle->realNumOfRows > 0;
return code; return code;
} }
...@@ -1456,11 +1439,11 @@ static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) { ...@@ -1456,11 +1439,11 @@ static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) {
return midPos; return midPos;
} }
static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end) { static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end) {
char* pData = NULL; char* pData = NULL;
int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1 : -1;
SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0];
TSKEY* tsArray = pCols->cols[0].pData; TSKEY* tsArray = pCols->cols[0].pData;
int32_t num = end - start + 1; int32_t num = end - start + 1;
...@@ -1470,12 +1453,12 @@ static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t c ...@@ -1470,12 +1453,12 @@ static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t c
return numOfRows; return numOfRows;
} }
int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns); int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns);
//data in buffer has greater timestamp, copy data in file block //data in buffer has greater timestamp, copy data in file block
int32_t i = 0, j = 0; int32_t i = 0, j = 0;
while(i < requiredNumOfCols && j < pCols->numOfCols) { while(i < requiredNumOfCols && j < pCols->numOfCols) {
SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
SDataCol* src = &pCols->cols[j]; SDataCol* src = &pCols->cols[j];
if (src->colId < pColInfo->info.colId) { if (src->colId < pColInfo->info.colId) {
...@@ -1485,7 +1468,7 @@ static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t c ...@@ -1485,7 +1468,7 @@ static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t c
int32_t bytes = pColInfo->info.bytes; int32_t bytes = pColInfo->info.bytes;
if (ASCENDING_TRAVERSE(pQueryHandle->order)) { if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;
} else { } else {
pData = (char*)pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes; pData = (char*)pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes;
...@@ -1523,8 +1506,8 @@ static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t c ...@@ -1523,8 +1506,8 @@ static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t c
} }
while (i < requiredNumOfCols) { // the remain columns are all null data while (i < requiredNumOfCols) { // the remain columns are all null data
SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
if (ASCENDING_TRAVERSE(pQueryHandle->order)) { if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;
} else { } else {
pData = (char*)pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes; pData = (char*)pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes;
...@@ -1544,15 +1527,15 @@ static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t c ...@@ -1544,15 +1527,15 @@ static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t c
i++; i++;
} }
pQueryHandle->cur.win.ekey = tsArray[end]; pTsdbReadHandle->cur.win.ekey = tsArray[end];
pQueryHandle->cur.lastKey = tsArray[end] + step; pTsdbReadHandle->cur.lastKey = tsArray[end] + step;
return numOfRows + num; return numOfRows + num;
} }
// Note: row1 always has high priority // Note: row1 always has high priority
static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows,
SMemRow row1, SMemRow row2, int32_t numOfCols, STable* pTable, SMemRow row1, SMemRow row2, int32_t numOfCols, uint64_t uid,
STSchema* pSchema1, STSchema* pSchema2, bool forceSetNull) { STSchema* pSchema1, STSchema* pSchema2, bool forceSetNull) {
char* pData = NULL; char* pData = NULL;
STSchema* pSchema; STSchema* pSchema;
...@@ -1570,8 +1553,9 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, ...@@ -1570,8 +1553,9 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity,
int32_t numOfColsOfRow1 = 0; int32_t numOfColsOfRow1 = 0;
if (pSchema1 == NULL) { if (pSchema1 == NULL) {
pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); pSchema1 = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, uid, 0);
} }
if(isRow1DataRow) { if(isRow1DataRow) {
numOfColsOfRow1 = schemaNCols(pSchema1); numOfColsOfRow1 = schemaNCols(pSchema1);
} else { } else {
...@@ -1582,7 +1566,7 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, ...@@ -1582,7 +1566,7 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity,
if(row2) { if(row2) {
isRow2DataRow = isDataRow(row2); isRow2DataRow = isDataRow(row2);
if (pSchema2 == NULL) { if (pSchema2 == NULL) {
pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); pSchema2 = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, uid, 0);
} }
if(isRow2DataRow) { if(isRow2DataRow) {
numOfColsOfRow2 = schemaNCols(pSchema2); numOfColsOfRow2 = schemaNCols(pSchema2);
...@@ -1594,9 +1578,9 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, ...@@ -1594,9 +1578,9 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity,
int32_t i = 0, j = 0, k = 0; int32_t i = 0, j = 0, k = 0;
while(i < numOfCols && (j < numOfColsOfRow1 || k < numOfColsOfRow2)) { while(i < numOfCols && (j < numOfColsOfRow1 || k < numOfColsOfRow2)) {
SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
if (ASCENDING_TRAVERSE(pQueryHandle->order)) { if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;
} else { } else {
pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes;
...@@ -1699,7 +1683,7 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, ...@@ -1699,7 +1683,7 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity,
SET_DOUBLE_PTR(pData, value); SET_DOUBLE_PTR(pData, value);
break; break;
case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_TIMESTAMP:
if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
*(TSKEY *)pData = tdGetKey(*(TKEY *)value); *(TSKEY *)pData = tdGetKey(*(TKEY *)value);
} else { } else {
*(TSKEY *)pData = *(TSKEY *)value; *(TSKEY *)pData = *(TSKEY *)value;
...@@ -1730,8 +1714,8 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, ...@@ -1730,8 +1714,8 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity,
if(forceSetNull) { if(forceSetNull) {
while (i < numOfCols) { // the remain columns are all null data while (i < numOfCols) { // the remain columns are all null data
SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
if (ASCENDING_TRAVERSE(pQueryHandle->order)) { if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;
} else { } else {
pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes;
...@@ -1748,29 +1732,29 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, ...@@ -1748,29 +1732,29 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity,
} }
} }
static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, int32_t numOfCols) { static void moveDataToFront(STsdbReadHandle* pTsdbReadHandle, int32_t numOfRows, int32_t numOfCols) {
if (numOfRows == 0 || ASCENDING_TRAVERSE(pQueryHandle->order)) { if (numOfRows == 0 || ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
return; return;
} }
// if the buffer is not full in case of descending order query, move the data in the front of the buffer // if the buffer is not full in case of descending order query, move the data in the front of the buffer
if (numOfRows < pQueryHandle->outputCapacity) { if (numOfRows < pTsdbReadHandle->outputCapacity) {
int32_t emptySize = pQueryHandle->outputCapacity - numOfRows; int32_t emptySize = pTsdbReadHandle->outputCapacity - numOfRows;
for(int32_t i = 0; i < numOfCols; ++i) { for(int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes);
} }
} }
} }
static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos, int32_t endPos, int32_t numOfExisted, static void getQualifiedRowsPos(STsdbReadHandle* pTsdbReadHandle, int32_t startPos, int32_t endPos, int32_t numOfExisted,
int32_t* start, int32_t* end) { int32_t* start, int32_t* end) {
*start = -1; *start = -1;
if (ASCENDING_TRAVERSE(pQueryHandle->order)) { if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
int32_t remain = endPos - startPos + 1; int32_t remain = endPos - startPos + 1;
if (remain + numOfExisted > pQueryHandle->outputCapacity) { if (remain + numOfExisted > pTsdbReadHandle->outputCapacity) {
*end = (pQueryHandle->outputCapacity - numOfExisted) + startPos - 1; *end = (pTsdbReadHandle->outputCapacity - numOfExisted) + startPos - 1;
} else { } else {
*end = endPos; *end = endPos;
} }
...@@ -1778,8 +1762,8 @@ static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos ...@@ -1778,8 +1762,8 @@ static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos
*start = startPos; *start = startPos;
} else { } else {
int32_t remain = (startPos - endPos) + 1; int32_t remain = (startPos - endPos) + 1;
if (remain + numOfExisted > pQueryHandle->outputCapacity) { if (remain + numOfExisted > pTsdbReadHandle->outputCapacity) {
*end = startPos + 1 - (pQueryHandle->outputCapacity - numOfExisted); *end = startPos + 1 - (pTsdbReadHandle->outputCapacity - numOfExisted);
} else { } else {
*end = endPos; *end = endPos;
} }
...@@ -1789,55 +1773,55 @@ static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos ...@@ -1789,55 +1773,55 @@ static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos
} }
} }
static void updateInfoAfterMerge(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, int32_t numOfRows, int32_t endPos) { static void updateInfoAfterMerge(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, int32_t numOfRows, int32_t endPos) {
SQueryFilePos* cur = &pQueryHandle->cur; SQueryFilePos* cur = &pTsdbReadHandle->cur;
pCheckInfo->lastKey = cur->lastKey; pCheckInfo->lastKey = cur->lastKey;
pQueryHandle->realNumOfRows = numOfRows; pTsdbReadHandle->realNumOfRows = numOfRows;
cur->rows = numOfRows; cur->rows = numOfRows;
cur->pos = endPos; cur->pos = endPos;
} }
static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle) { static void doCheckGeneratedBlockRange(STsdbReadHandle* pTsdbReadHandle) {
SQueryFilePos* cur = &pQueryHandle->cur; SQueryFilePos* cur = &pTsdbReadHandle->cur;
if (cur->rows > 0) { if (cur->rows > 0) {
if (ASCENDING_TRAVERSE(pQueryHandle->order)) { if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
assert(cur->win.skey >= pQueryHandle->window.skey && cur->win.ekey <= pQueryHandle->window.ekey); assert(cur->win.skey >= pTsdbReadHandle->window.skey && cur->win.ekey <= pTsdbReadHandle->window.ekey);
} else { } else {
assert(cur->win.skey >= pQueryHandle->window.ekey && cur->win.ekey <= pQueryHandle->window.skey); assert(cur->win.skey >= pTsdbReadHandle->window.ekey && cur->win.ekey <= pTsdbReadHandle->window.skey);
} }
SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, 0); SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, 0);
assert(cur->win.skey == ((TSKEY*)pColInfoData->pData)[0] && cur->win.ekey == ((TSKEY*)pColInfoData->pData)[cur->rows-1]); assert(cur->win.skey == ((TSKEY*)pColInfoData->pData)[0] && cur->win.ekey == ((TSKEY*)pColInfoData->pData)[cur->rows-1]);
} else { } else {
cur->win = pQueryHandle->window; cur->win = pTsdbReadHandle->window;
int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1:-1;
cur->lastKey = pQueryHandle->window.ekey + step; cur->lastKey = pTsdbReadHandle->window.ekey + step;
} }
} }
static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos) { static void copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos) {
SQueryFilePos* cur = &pQueryHandle->cur; SQueryFilePos* cur = &pTsdbReadHandle->cur;
SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0];
TSKEY* tsArray = pCols->cols[0].pData; TSKEY* tsArray = pCols->cols[0].pData;
int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1:-1;
int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pTsdbReadHandle));
int32_t pos = cur->pos; int32_t pos = cur->pos;
int32_t start = cur->pos; int32_t start = cur->pos;
int32_t end = endPos; int32_t end = endPos;
if (!ASCENDING_TRAVERSE(pQueryHandle->order)) { if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
SWAP(start, end, int32_t); SWAP(start, end, int32_t);
} }
assert(pQueryHandle->outputCapacity >= (end - start + 1)); assert(pTsdbReadHandle->outputCapacity >= (end - start + 1));
int32_t numOfRows = doCopyRowsFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, 0, start, end); int32_t numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, 0, start, end);
// the time window should always be ascending order: skey <= ekey // the time window should always be ascending order: skey <= ekey
cur->win = (STimeWindow) {.skey = tsArray[start], .ekey = tsArray[end]}; cur->win = (STimeWindow) {.skey = tsArray[start], .ekey = tsArray[end]};
...@@ -1846,35 +1830,34 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl ...@@ -1846,35 +1830,34 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl
cur->blockCompleted = true; cur->blockCompleted = true;
// if the buffer is not full in case of descending order query, move the data in the front of the buffer // if the buffer is not full in case of descending order query, move the data in the front of the buffer
moveDataToFront(pQueryHandle, numOfRows, numOfCols); moveDataToFront(pTsdbReadHandle, numOfRows, numOfCols);
// The value of pos may be -1 or pBlockInfo->rows, and it is invalid in both cases. // The value of pos may be -1 or pBlockInfo->rows, and it is invalid in both cases.
pos = endPos + step; pos = endPos + step;
updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos); updateInfoAfterMerge(pTsdbReadHandle, pCheckInfo, numOfRows, pos);
doCheckGeneratedBlockRange(pQueryHandle); doCheckGeneratedBlockRange(pTsdbReadHandle);
tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, 0x%"PRIx64, tsdbDebug("%p uid:%" PRIu64", data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, 0x%"PRIx64,
pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->mixBlock, cur->win.skey, pTsdbReadHandle, pCheckInfo->tableId, cur->mixBlock, cur->win.skey, cur->win.ekey, cur->rows, pTsdbReadHandle->qId);
cur->win.ekey, cur->rows, pQueryHandle->qId);
} }
int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBlockInfo) { int32_t getEndPosInDataBlock(STsdbReadHandle* pTsdbReadHandle, SDataBlockInfo* pBlockInfo) {
// NOTE: reverse the order to find the end position in data block // NOTE: reverse the order to find the end position in data block
int32_t endPos = -1; int32_t endPos = -1;
int32_t order = ASCENDING_TRAVERSE(pQueryHandle->order)? TSDB_ORDER_DESC : TSDB_ORDER_ASC; int32_t order = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? TSDB_ORDER_DESC : TSDB_ORDER_ASC;
SQueryFilePos* cur = &pQueryHandle->cur; SQueryFilePos* cur = &pTsdbReadHandle->cur;
SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0];
if (ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey >= pBlockInfo->window.ekey) { if (ASCENDING_TRAVERSE(pTsdbReadHandle->order) && pTsdbReadHandle->window.ekey >= pBlockInfo->window.ekey) {
endPos = pBlockInfo->rows - 1; endPos = pBlockInfo->rows - 1;
cur->mixBlock = (cur->pos != 0); cur->mixBlock = (cur->pos != 0);
} else if (!ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey <= pBlockInfo->window.skey) { } else if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && pTsdbReadHandle->window.ekey <= pBlockInfo->window.skey) {
endPos = 0; endPos = 0;
cur->mixBlock = (cur->pos != pBlockInfo->rows - 1); cur->mixBlock = (cur->pos != pBlockInfo->rows - 1);
} else { } else {
assert(pCols->numOfRows > 0); assert(pCols->numOfRows > 0);
endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pQueryHandle->window.ekey, order); endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pTsdbReadHandle->window.ekey, order);
cur->mixBlock = true; cur->mixBlock = true;
} }
...@@ -1883,33 +1866,33 @@ int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBl ...@@ -1883,33 +1866,33 @@ int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBl
// only return the qualified data to client in terms of query time window, data rows in the same block but do not // only return the qualified data to client in terms of query time window, data rows in the same block but do not
// be included in the query time window will be discarded // be included in the query time window will be discarded
static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock) { static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock) {
SQueryFilePos* cur = &pQueryHandle->cur; SQueryFilePos* cur = &pTsdbReadHandle->cur;
SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); SDataBlockInfo blockInfo = {0};//GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock);
STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config;
initTableMemIterator(pQueryHandle, pCheckInfo); initTableMemIterator(pTsdbReadHandle, pCheckInfo);
SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0];
assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == PRIMARYKEY_TIMESTAMP_COL_INDEX && assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == PRIMARYKEY_TIMESTAMP_COL_ID &&
cur->pos >= 0 && cur->pos < pBlock->numOfRows); cur->pos >= 0 && cur->pos < pBlock->numOfRows);
TSKEY* tsArray = pCols->cols[0].pData; TSKEY* tsArray = pCols->cols[0].pData;
assert(pCols->numOfRows == pBlock->numOfRows && tsArray[0] == pBlock->keyFirst && tsArray[pBlock->numOfRows-1] == pBlock->keyLast); assert(pCols->numOfRows == pBlock->numOfRows && tsArray[0] == pBlock->keyFirst && tsArray[pBlock->numOfRows-1] == pBlock->keyLast);
// for search the endPos, so the order needs to reverse // for search the endPos, so the order needs to reverse
int32_t order = (pQueryHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; int32_t order = (pTsdbReadHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC;
int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1:-1;
int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pTsdbReadHandle));
STable* pTable = pCheckInfo->pTableObj; STable* pTable = pCheckInfo->pTableObj;
int32_t endPos = getEndPosInDataBlock(pQueryHandle, &blockInfo); int32_t endPos = getEndPosInDataBlock(pTsdbReadHandle, &blockInfo);
tsdbDebug("%p uid:%" PRIu64",tid:%d start merge data block, file block range:%"PRIu64"-%"PRIu64" rows:%d, start:%d," tsdbDebug("%p uid:%" PRIu64" start merge data block, file block range:%"PRIu64"-%"PRIu64" rows:%d, start:%d,"
"end:%d, 0x%"PRIx64, "end:%d, 0x%"PRIx64,
pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, blockInfo.window.skey, blockInfo.window.ekey, pTsdbReadHandle, pCheckInfo->tableId, blockInfo.window.skey, blockInfo.window.ekey,
blockInfo.rows, cur->pos, endPos, pQueryHandle->qId); blockInfo.rows, cur->pos, endPos, pTsdbReadHandle->qId);
// compared with the data from in-memory buffer, to generate the correct timestamp array list // compared with the data from in-memory buffer, to generate the correct timestamp array list
int32_t numOfRows = 0; int32_t numOfRows = 0;
...@@ -1924,40 +1907,40 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* ...@@ -1924,40 +1907,40 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
// no data in buffer, load data from file directly // no data in buffer, load data from file directly
if (pCheckInfo->iiter == NULL && pCheckInfo->iter == NULL) { if (pCheckInfo->iiter == NULL && pCheckInfo->iter == NULL) {
copyAllRemainRowsFromFileBlock(pQueryHandle, pCheckInfo, &blockInfo, endPos); copyAllRemainRowsFromFileBlock(pTsdbReadHandle, pCheckInfo, &blockInfo, endPos);
return; return;
} else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) { } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) {
SSkipListNode* node = NULL; SSkipListNode* node = NULL;
do { do {
SMemRow row2 = NULL; SMemRow row2 = NULL;
SMemRow row1 = getSMemRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update, &row2); SMemRow row1 = getSMemRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, &row2);
if (row1 == NULL) { if (row1 == NULL) {
break; break;
} }
TSKEY key = memRowKey(row1); TSKEY key = memRowKey(row1);
if ((key > pQueryHandle->window.ekey && ASCENDING_TRAVERSE(pQueryHandle->order)) || if ((key > pTsdbReadHandle->window.ekey && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) ||
(key < pQueryHandle->window.ekey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { (key < pTsdbReadHandle->window.ekey && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) {
break; break;
} }
if (((pos > endPos || tsArray[pos] > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || if (((pos > endPos || tsArray[pos] > pTsdbReadHandle->window.ekey) && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) ||
((pos < endPos || tsArray[pos] < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) { ((pos < endPos || tsArray[pos] < pTsdbReadHandle->window.ekey) && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) {
break; break;
} }
if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) ||
(key > tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { (key > tsArray[pos] && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) {
if (rv1 != memRowVersion(row1)) { if (rv1 != memRowVersion(row1)) {
pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); // pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1));
rv1 = memRowVersion(row1); rv1 = memRowVersion(row1);
} }
if(row2 && rv2 != memRowVersion(row2)) { if(row2 && rv2 != memRowVersion(row2)) {
pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); // pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2));
rv2 = memRowVersion(row2); rv2 = memRowVersion(row2);
} }
mergeTwoRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row1, row2, numOfCols, pTable, pSchema1, pSchema2, true); mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, row1, row2, numOfCols, pTable, pSchema1, pSchema2, true);
numOfRows += 1; numOfRows += 1;
if (cur->win.skey == TSKEY_INITIAL_VAL) { if (cur->win.skey == TSKEY_INITIAL_VAL) {
cur->win.skey = key; cur->win.skey = key;
...@@ -1971,19 +1954,19 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* ...@@ -1971,19 +1954,19 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
} else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it } else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it
if (pCfg->update) { if (pCfg->update) {
if(pCfg->update == TD_ROW_PARTIAL_UPDATE) { if(pCfg->update == TD_ROW_PARTIAL_UPDATE) {
doCopyRowsFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, pos, pos); doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, pos, pos);
} }
if (rv1 != memRowVersion(row1)) { if (rv1 != memRowVersion(row1)) {
pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); // pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1));
rv1 = memRowVersion(row1); rv1 = memRowVersion(row1);
} }
if(row2 && rv2 != memRowVersion(row2)) { if(row2 && rv2 != memRowVersion(row2)) {
pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); // pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2));
rv2 = memRowVersion(row2); rv2 = memRowVersion(row2);
} }
bool forceSetNull = pCfg->update != TD_ROW_PARTIAL_UPDATE; bool forceSetNull = pCfg->update != TD_ROW_PARTIAL_UPDATE;
mergeTwoRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row1, row2, numOfCols, pTable, pSchema1, pSchema2, forceSetNull); mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, row1, row2, numOfCols, pTable, pSchema1, pSchema2, forceSetNull);
numOfRows += 1; numOfRows += 1;
if (cur->win.skey == TSKEY_INITIAL_VAL) { if (cur->win.skey == TSKEY_INITIAL_VAL) {
cur->win.skey = key; cur->win.skey = key;
...@@ -1998,8 +1981,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* ...@@ -1998,8 +1981,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
} else { } else {
moveToNextRowInMem(pCheckInfo); moveToNextRowInMem(pCheckInfo);
} }
} else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || } else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) ||
(key < tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { (key < tsArray[pos] && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) {
if (cur->win.skey == TSKEY_INITIAL_VAL) { if (cur->win.skey == TSKEY_INITIAL_VAL) {
cur->win.skey = tsArray[pos]; cur->win.skey = tsArray[pos];
} }
...@@ -2016,38 +1999,38 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* ...@@ -2016,38 +1999,38 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
} }
int32_t qstart = 0, qend = 0; int32_t qstart = 0, qend = 0;
getQualifiedRowsPos(pQueryHandle, pos, end, numOfRows, &qstart, &qend); getQualifiedRowsPos(pTsdbReadHandle, pos, end, numOfRows, &qstart, &qend);
numOfRows = doCopyRowsFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, qstart, qend); numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, qstart, qend);
pos += (qend - qstart + 1) * step; pos += (qend - qstart + 1) * step;
cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? tsArray[qend]:tsArray[qstart]; cur->win.ekey = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? tsArray[qend]:tsArray[qstart];
cur->lastKey = cur->win.ekey + step; cur->lastKey = cur->win.ekey + step;
} }
} while (numOfRows < pQueryHandle->outputCapacity); } while (numOfRows < pTsdbReadHandle->outputCapacity);
if (numOfRows < pQueryHandle->outputCapacity) { if (numOfRows < pTsdbReadHandle->outputCapacity) {
/** /**
* if cache is empty, load remain file block data. In contrast, if there are remain data in cache, do NOT * if cache is empty, load remain file block data. In contrast, if there are remain data in cache, do NOT
* copy them all to result buffer, since it may be overlapped with file data block. * copy them all to result buffer, since it may be overlapped with file data block.
*/ */
if (node == NULL || if (node == NULL ||
((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && ((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) > pTsdbReadHandle->window.ekey) &&
ASCENDING_TRAVERSE(pQueryHandle->order)) || ASCENDING_TRAVERSE(pTsdbReadHandle->order)) ||
((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && ((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) < pTsdbReadHandle->window.ekey) &&
!ASCENDING_TRAVERSE(pQueryHandle->order))) { !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) {
// no data in cache or data in cache is greater than the ekey of time window, load data from file block // no data in cache or data in cache is greater than the ekey of time window, load data from file block
if (cur->win.skey == TSKEY_INITIAL_VAL) { if (cur->win.skey == TSKEY_INITIAL_VAL) {
cur->win.skey = tsArray[pos]; cur->win.skey = tsArray[pos];
} }
int32_t start = -1, end = -1; int32_t start = -1, end = -1;
getQualifiedRowsPos(pQueryHandle, pos, endPos, numOfRows, &start, &end); getQualifiedRowsPos(pTsdbReadHandle, pos, endPos, numOfRows, &start, &end);
numOfRows = doCopyRowsFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end); numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, start, end);
pos += (end - start + 1) * step; pos += (end - start + 1) * step;
cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? tsArray[end]:tsArray[start]; cur->win.ekey = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? tsArray[end]:tsArray[start];
cur->lastKey = cur->win.ekey + step; cur->lastKey = cur->win.ekey + step;
cur->mixBlock = true; cur->mixBlock = true;
} }
...@@ -2055,20 +2038,19 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* ...@@ -2055,20 +2038,19 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
} }
cur->blockCompleted = cur->blockCompleted =
(((pos > endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || (((pos > endPos || cur->lastKey > pTsdbReadHandle->window.ekey) && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) ||
((pos < endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))); ((pos < endPos || cur->lastKey < pTsdbReadHandle->window.ekey) && !ASCENDING_TRAVERSE(pTsdbReadHandle->order)));
if (!ASCENDING_TRAVERSE(pQueryHandle->order)) { if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
SWAP(cur->win.skey, cur->win.ekey, TSKEY); SWAP(cur->win.skey, cur->win.ekey, TSKEY);
} }
moveDataToFront(pQueryHandle, numOfRows, numOfCols); moveDataToFront(pTsdbReadHandle, numOfRows, numOfCols);
updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos); updateInfoAfterMerge(pTsdbReadHandle, pCheckInfo, numOfRows, pos);
doCheckGeneratedBlockRange(pQueryHandle); doCheckGeneratedBlockRange(pTsdbReadHandle);
tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, 0x%"PRIx64, tsdbDebug("%p uid:%" PRIu64", data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, 0x%"PRIx64,
pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->mixBlock, cur->win.skey, pTsdbReadHandle, pCheckInfo->tableId, cur->mixBlock, cur->win.skey, cur->win.ekey, cur->rows, pTsdbReadHandle->qId);
cur->win.ekey, cur->rows, pQueryHandle->qId);
} }
int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) {
...@@ -2174,24 +2156,24 @@ static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void* ...@@ -2174,24 +2156,24 @@ static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void*
return pLeftBlockInfoEx->compBlock->offset > pRightBlockInfoEx->compBlock->offset ? 1 : -1; return pLeftBlockInfoEx->compBlock->offset > pRightBlockInfoEx->compBlock->offset ? 1 : -1;
} }
static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numOfBlocks, int32_t* numOfAllocBlocks) { static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t numOfBlocks, int32_t* numOfAllocBlocks) {
size_t size = sizeof(STableBlockInfo) * numOfBlocks; size_t size = sizeof(STableBlockInfo) * numOfBlocks;
if (pQueryHandle->allocSize < size) { if (pTsdbReadHandle->allocSize < size) {
pQueryHandle->allocSize = (int32_t)size; pTsdbReadHandle->allocSize = (int32_t)size;
char* tmp = realloc(pQueryHandle->pDataBlockInfo, pQueryHandle->allocSize); char* tmp = realloc(pTsdbReadHandle->pDataBlockInfo, pTsdbReadHandle->allocSize);
if (tmp == NULL) { if (tmp == NULL) {
return TSDB_CODE_TDB_OUT_OF_MEMORY; return TSDB_CODE_TDB_OUT_OF_MEMORY;
} }
pQueryHandle->pDataBlockInfo = (STableBlockInfo*) tmp; pTsdbReadHandle->pDataBlockInfo = (STableBlockInfo*) tmp;
} }
memset(pQueryHandle->pDataBlockInfo, 0, size); memset(pTsdbReadHandle->pDataBlockInfo, 0, size);
*numOfAllocBlocks = numOfBlocks; *numOfAllocBlocks = numOfBlocks;
// access data blocks according to the offset of each block in asc/desc order. // access data blocks according to the offset of each block in asc/desc order.
int32_t numOfTables = (int32_t)taosArrayGetSize(pQueryHandle->pTableCheckInfo); int32_t numOfTables = (int32_t)taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo);
SBlockOrderSupporter sup = {0}; SBlockOrderSupporter sup = {0};
sup.numOfTables = numOfTables; sup.numOfTables = numOfTables;
...@@ -2208,7 +2190,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO ...@@ -2208,7 +2190,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
int32_t numOfQualTables = 0; int32_t numOfQualTables = 0;
for (int32_t j = 0; j < numOfTables; ++j) { for (int32_t j = 0; j < numOfTables; ++j) {
STableCheckInfo* pTableCheck = (STableCheckInfo*)taosArrayGet(pQueryHandle->pTableCheckInfo, j); STableCheckInfo* pTableCheck = (STableCheckInfo*)taosArrayGet(pTsdbReadHandle->pTableCheckInfo, j);
if (pTableCheck->numOfBlocks <= 0) { if (pTableCheck->numOfBlocks <= 0) {
continue; continue;
} }
...@@ -2239,16 +2221,16 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO ...@@ -2239,16 +2221,16 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
// since there is only one table qualified, blocks are not sorted // since there is only one table qualified, blocks are not sorted
if (numOfQualTables == 1) { if (numOfQualTables == 1) {
memcpy(pQueryHandle->pDataBlockInfo, sup.pDataBlockInfo[0], sizeof(STableBlockInfo) * numOfBlocks); memcpy(pTsdbReadHandle->pDataBlockInfo, sup.pDataBlockInfo[0], sizeof(STableBlockInfo) * numOfBlocks);
cleanBlockOrderSupporter(&sup, numOfQualTables); cleanBlockOrderSupporter(&sup, numOfQualTables);
tsdbDebug("%p create data blocks info struct completed for 1 table, %d blocks not sorted 0x%"PRIx64, pQueryHandle, cnt, tsdbDebug("%p create data blocks info struct completed for 1 table, %d blocks not sorted 0x%"PRIx64, pTsdbReadHandle, cnt,
pQueryHandle->qId); pTsdbReadHandle->qId);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables 0x%"PRIx64, pQueryHandle, cnt, tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables 0x%"PRIx64, pTsdbReadHandle, cnt,
numOfQualTables, pQueryHandle->qId); numOfQualTables, pTsdbReadHandle->qId);
assert(cnt <= numOfBlocks && numOfQualTables <= numOfTables); // the pTableQueryInfo[j]->numOfBlocks may be 0 assert(cnt <= numOfBlocks && numOfQualTables <= numOfTables); // the pTableQueryInfo[j]->numOfBlocks may be 0
sup.numOfTables = numOfQualTables; sup.numOfTables = numOfQualTables;
...@@ -2267,7 +2249,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO ...@@ -2267,7 +2249,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
int32_t index = sup.blockIndexArray[pos]++; int32_t index = sup.blockIndexArray[pos]++;
STableBlockInfo* pBlocksInfo = sup.pDataBlockInfo[pos]; STableBlockInfo* pBlocksInfo = sup.pDataBlockInfo[pos];
pQueryHandle->pDataBlockInfo[numOfTotal++] = pBlocksInfo[index]; pTsdbReadHandle->pDataBlockInfo[numOfTotal++] = pBlocksInfo[index];
// set data block index overflow, in order to disable the offset comparator // set data block index overflow, in order to disable the offset comparator
if (sup.blockIndexArray[pos] >= sup.numOfBlocksPerTable[pos]) { if (sup.blockIndexArray[pos] >= sup.numOfBlocksPerTable[pos]) {
...@@ -2284,90 +2266,90 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO ...@@ -2284,90 +2266,90 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
* } * }
*/ */
tsdbDebug("%p %d data blocks sort completed, 0x%"PRIx64, pQueryHandle, cnt, pQueryHandle->qId); tsdbDebug("%p %d data blocks sort completed, 0x%"PRIx64, pTsdbReadHandle, cnt, pTsdbReadHandle->qId);
cleanBlockOrderSupporter(&sup, numOfTables); cleanBlockOrderSupporter(&sup, numOfTables);
free(pTree); free(pTree);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exists); static int32_t getFirstFileDataBlock(STsdbReadHandle* pTsdbReadHandle, bool* exists);
static int32_t getDataBlockRv(STsdbQueryHandle* pQueryHandle, STableBlockInfo* pNext, bool *exists) { static int32_t getDataBlockRv(STsdbReadHandle* pTsdbReadHandle, STableBlockInfo* pNext, bool *exists) {
int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1 : -1;
SQueryFilePos* cur = &pQueryHandle->cur; SQueryFilePos* cur = &pTsdbReadHandle->cur;
while(1) { while(1) {
int32_t code = loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo, exists); int32_t code = loadFileDataBlock(pTsdbReadHandle, pNext->compBlock, pNext->pTableCheckInfo, exists);
if (code != TSDB_CODE_SUCCESS || *exists) { if (code != TSDB_CODE_SUCCESS || *exists) {
return code; return code;
} }
if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pQueryHandle->order)) || if ((cur->slot == pTsdbReadHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) ||
(cur->slot == 0 && !ASCENDING_TRAVERSE(pQueryHandle->order))) { (cur->slot == 0 && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) {
// all data blocks in current file has been checked already, try next file if exists // all data blocks in current file has been checked already, try next file if exists
return getFirstFileDataBlock(pQueryHandle, exists); return getFirstFileDataBlock(pTsdbReadHandle, exists);
} else { // next block of the same file } else { // next block of the same file
cur->slot += step; cur->slot += step;
cur->mixBlock = false; cur->mixBlock = false;
cur->blockCompleted = false; cur->blockCompleted = false;
pNext = &pQueryHandle->pDataBlockInfo[cur->slot]; pNext = &pTsdbReadHandle->pDataBlockInfo[cur->slot];
} }
} }
} }
static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exists) { static int32_t getFirstFileDataBlock(STsdbReadHandle* pTsdbReadHandle, bool* exists) {
pQueryHandle->numOfBlocks = 0; pTsdbReadHandle->numOfBlocks = 0;
SQueryFilePos* cur = &pQueryHandle->cur; SQueryFilePos* cur = &pTsdbReadHandle->cur;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t numOfBlocks = 0; int32_t numOfBlocks = 0;
int32_t numOfTables = (int32_t)taosArrayGetSize(pQueryHandle->pTableCheckInfo); int32_t numOfTables = (int32_t)taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo);
STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config;
STimeWindow win = TSWINDOW_INITIALIZER; STimeWindow win = TSWINDOW_INITIALIZER;
while (true) { while (true) {
tsdbRLockFS(REPO_FS(pQueryHandle->pTsdb)); tsdbRLockFS(REPO_FS(pTsdbReadHandle->pTsdb));
if ((pQueryHandle->pFileGroup = tsdbFSIterNext(&pQueryHandle->fileIter)) == NULL) { if ((pTsdbReadHandle->pFileGroup = tsdbFSIterNext(&pTsdbReadHandle->fileIter)) == NULL) {
tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb));
break; break;
} }
tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pQueryHandle->pFileGroup->fid, &win.skey, &win.ekey); tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pTsdbReadHandle->pFileGroup->fid, &win.skey, &win.ekey);
// current file are not overlapped with query time window, ignore remain files // current file are not overlapped with query time window, ignore remain files
if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) || if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.skey > pTsdbReadHandle->window.ekey) ||
(!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) { (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.ekey < pTsdbReadHandle->window.ekey)) {
tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb));
tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pQueryHandle, tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pTsdbReadHandle,
pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qId); pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, pTsdbReadHandle->qId);
pQueryHandle->pFileGroup = NULL; pTsdbReadHandle->pFileGroup = NULL;
assert(pQueryHandle->numOfBlocks == 0); assert(pTsdbReadHandle->numOfBlocks == 0);
break; break;
} }
if (tsdbSetAndOpenReadFSet(&pQueryHandle->rhelper, pQueryHandle->pFileGroup) < 0) { if (tsdbSetAndOpenReadFSet(&pTsdbReadHandle->rhelper, pTsdbReadHandle->pFileGroup) < 0) {
tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb));
code = terrno; code = terrno;
break; break;
} }
tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb));
if (tsdbLoadBlockIdx(&pQueryHandle->rhelper) < 0) { if (tsdbLoadBlockIdx(&pTsdbReadHandle->rhelper) < 0) {
code = terrno; code = terrno;
break; break;
} }
if ((code = getFileCompInfo(pQueryHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) { if ((code = getFileCompInfo(pTsdbReadHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) {
break; break;
} }
tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, 0x%"PRIx64, pQueryHandle, numOfBlocks, numOfTables, tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, 0x%"PRIx64, pTsdbReadHandle, numOfBlocks, numOfTables,
pQueryHandle->pFileGroup->fid, pQueryHandle->qId); pTsdbReadHandle->pFileGroup->fid, pTsdbReadHandle->qId);
assert(numOfBlocks >= 0); assert(numOfBlocks >= 0);
if (numOfBlocks == 0) { if (numOfBlocks == 0) {
...@@ -2375,20 +2357,20 @@ static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exist ...@@ -2375,20 +2357,20 @@ static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exist
} }
// todo return error code to query engine // todo return error code to query engine
if ((code = createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks)) != TSDB_CODE_SUCCESS) { if ((code = createDataBlocksInfo(pTsdbReadHandle, numOfBlocks, &pTsdbReadHandle->numOfBlocks)) != TSDB_CODE_SUCCESS) {
break; break;
} }
assert(numOfBlocks >= pQueryHandle->numOfBlocks); assert(numOfBlocks >= pTsdbReadHandle->numOfBlocks);
if (pQueryHandle->numOfBlocks > 0) { if (pTsdbReadHandle->numOfBlocks > 0) {
break; break;
} }
} }
// no data in file anymore // no data in file anymore
if (pQueryHandle->numOfBlocks <= 0 || code != TSDB_CODE_SUCCESS) { if (pTsdbReadHandle->numOfBlocks <= 0 || code != TSDB_CODE_SUCCESS) {
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
assert(pQueryHandle->pFileGroup == NULL); assert(pTsdbReadHandle->pFileGroup == NULL);
} }
cur->fid = INT32_MIN; // denote that there are no data in file anymore cur->fid = INT32_MIN; // denote that there are no data in file anymore
...@@ -2396,12 +2378,12 @@ static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exist ...@@ -2396,12 +2378,12 @@ static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exist
return code; return code;
} }
assert(pQueryHandle->pFileGroup != NULL && pQueryHandle->numOfBlocks > 0); assert(pTsdbReadHandle->pFileGroup != NULL && pTsdbReadHandle->numOfBlocks > 0);
cur->slot = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1; cur->slot = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 0:pTsdbReadHandle->numOfBlocks-1;
cur->fid = pQueryHandle->pFileGroup->fid; cur->fid = pTsdbReadHandle->pFileGroup->fid;
STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; STableBlockInfo* pBlockInfo = &pTsdbReadHandle->pDataBlockInfo[cur->slot];
return getDataBlockRv(pQueryHandle, pBlockInfo, exists); return getDataBlockRv(pTsdbReadHandle, pBlockInfo, exists);
} }
static bool isEndFileDataBlock(SQueryFilePos* cur, int32_t numOfBlocks, bool ascTrav) { static bool isEndFileDataBlock(SQueryFilePos* cur, int32_t numOfBlocks, bool ascTrav) {
...@@ -2409,90 +2391,90 @@ static bool isEndFileDataBlock(SQueryFilePos* cur, int32_t numOfBlocks, bool asc ...@@ -2409,90 +2391,90 @@ static bool isEndFileDataBlock(SQueryFilePos* cur, int32_t numOfBlocks, bool asc
return (cur->slot == numOfBlocks - 1 && ascTrav) || (cur->slot == 0 && !ascTrav); return (cur->slot == numOfBlocks - 1 && ascTrav) || (cur->slot == 0 && !ascTrav);
} }
static void moveToNextDataBlockInCurrentFile(STsdbQueryHandle* pQueryHandle) { static void moveToNextDataBlockInCurrentFile(STsdbReadHandle* pTsdbReadHandle) {
int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1 : -1;
SQueryFilePos* cur = &pQueryHandle->cur; SQueryFilePos* cur = &pTsdbReadHandle->cur;
assert(cur->slot < pQueryHandle->numOfBlocks && cur->slot >= 0); assert(cur->slot < pTsdbReadHandle->numOfBlocks && cur->slot >= 0);
cur->slot += step; cur->slot += step;
cur->mixBlock = false; cur->mixBlock = false;
cur->blockCompleted = false; cur->blockCompleted = false;
} }
#if 0
int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist* pTableBlockInfo) { int32_t tsdbGetFileBlocksDistInfo(tsdbReadHandleT* queryHandle, STableBlockDist* pTableBlockInfo) {
STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) queryHandle; STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) queryHandle;
pTableBlockInfo->totalSize = 0; pTableBlockInfo->totalSize = 0;
pTableBlockInfo->totalRows = 0; pTableBlockInfo->totalRows = 0;
STsdbFS* pFileHandle = REPO_FS(pQueryHandle->pTsdb); STsdbFS* pFileHandle = REPO_FS(pTsdbReadHandle->pTsdb);
// find the start data block in file // find the start data block in file
pQueryHandle->locateStart = true; pTsdbReadHandle->locateStart = true;
STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config;
int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision); int32_t fid = getFileIdFromKey(pTsdbReadHandle->window.skey, pCfg->daysPerFile, pCfg->precision);
tsdbRLockFS(pFileHandle); tsdbRLockFS(pFileHandle);
tsdbFSIterInit(&pQueryHandle->fileIter, pFileHandle, pQueryHandle->order); tsdbFSIterInit(&pTsdbReadHandle->fileIter, pFileHandle, pTsdbReadHandle->order);
tsdbFSIterSeek(&pQueryHandle->fileIter, fid); tsdbFSIterSeek(&pTsdbReadHandle->fileIter, fid);
tsdbUnLockFS(pFileHandle); tsdbUnLockFS(pFileHandle);
pTableBlockInfo->numOfFiles += 1; pTableBlockInfo->numOfFiles += 1;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t numOfBlocks = 0; int32_t numOfBlocks = 0;
int32_t numOfTables = (int32_t)taosArrayGetSize(pQueryHandle->pTableCheckInfo); int32_t numOfTables = (int32_t)taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo);
int defaultRows = TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock); int defaultRows = TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock);
STimeWindow win = TSWINDOW_INITIALIZER; STimeWindow win = TSWINDOW_INITIALIZER;
while (true) { while (true) {
numOfBlocks = 0; numOfBlocks = 0;
tsdbRLockFS(REPO_FS(pQueryHandle->pTsdb)); tsdbRLockFS(REPO_FS(pTsdbReadHandle->pTsdb));
if ((pQueryHandle->pFileGroup = tsdbFSIterNext(&pQueryHandle->fileIter)) == NULL) { if ((pTsdbReadHandle->pFileGroup = tsdbFSIterNext(&pTsdbReadHandle->fileIter)) == NULL) {
tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb));
break; break;
} }
tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pQueryHandle->pFileGroup->fid, &win.skey, &win.ekey); tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pTsdbReadHandle->pFileGroup->fid, &win.skey, &win.ekey);
// current file are not overlapped with query time window, ignore remain files // current file are not overlapped with query time window, ignore remain files
if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) || if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.skey > pTsdbReadHandle->window.ekey) ||
(!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) { (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.ekey < pTsdbReadHandle->window.ekey)) {
tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb));
tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pQueryHandle, tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pTsdbReadHandle,
pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qId); pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, pTsdbReadHandle->qId);
pQueryHandle->pFileGroup = NULL; pTsdbReadHandle->pFileGroup = NULL;
break; break;
} }
pTableBlockInfo->numOfFiles += 1; pTableBlockInfo->numOfFiles += 1;
if (tsdbSetAndOpenReadFSet(&pQueryHandle->rhelper, pQueryHandle->pFileGroup) < 0) { if (tsdbSetAndOpenReadFSet(&pTsdbReadHandle->rhelper, pTsdbReadHandle->pFileGroup) < 0) {
tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb));
code = terrno; code = terrno;
break; break;
} }
tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb));
if (tsdbLoadBlockIdx(&pQueryHandle->rhelper) < 0) { if (tsdbLoadBlockIdx(&pTsdbReadHandle->rhelper) < 0) {
code = terrno; code = terrno;
break; break;
} }
if ((code = getFileCompInfo(pQueryHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) { if ((code = getFileCompInfo(pTsdbReadHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) {
break; break;
} }
tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, 0x%"PRIx64, pQueryHandle, numOfBlocks, numOfTables, tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, 0x%"PRIx64, pTsdbReadHandle, numOfBlocks, numOfTables,
pQueryHandle->pFileGroup->fid, pQueryHandle->qId); pTsdbReadHandle->pFileGroup->fid, pTsdbReadHandle->qId);
if (numOfBlocks == 0) { if (numOfBlocks == 0) {
continue; continue;
} }
for (int32_t i = 0; i < numOfTables; ++i) { for (int32_t i = 0; i < numOfTables; ++i) {
STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i);
SBlock* pBlock = pCheckInfo->pCompInfo->blocks; SBlock* pBlock = pCheckInfo->pCompInfo->blocks;
for (int32_t j = 0; j < pCheckInfo->numOfBlocks; ++j) { for (int32_t j = 0; j < pCheckInfo->numOfBlocks; ++j) {
...@@ -2512,36 +2494,37 @@ int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist ...@@ -2512,36 +2494,37 @@ int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist
return code; return code;
} }
#endif
static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists) { static int32_t getDataBlocksInFiles(STsdbReadHandle* pTsdbReadHandle, bool* exists) {
STsdbFS* pFileHandle = REPO_FS(pQueryHandle->pTsdb); STsdbFS* pFileHandle = REPO_FS(pTsdbReadHandle->pTsdb);
SQueryFilePos* cur = &pQueryHandle->cur; SQueryFilePos* cur = &pTsdbReadHandle->cur;
// find the start data block in file // find the start data block in file
if (!pQueryHandle->locateStart) { if (!pTsdbReadHandle->locateStart) {
pQueryHandle->locateStart = true; pTsdbReadHandle->locateStart = true;
STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config;
int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision); int32_t fid = getFileIdFromKey(pTsdbReadHandle->window.skey, pCfg->daysPerFile, pCfg->precision);
tsdbRLockFS(pFileHandle); tsdbRLockFS(pFileHandle);
tsdbFSIterInit(&pQueryHandle->fileIter, pFileHandle, pQueryHandle->order); tsdbFSIterInit(&pTsdbReadHandle->fileIter, pFileHandle, pTsdbReadHandle->order);
tsdbFSIterSeek(&pQueryHandle->fileIter, fid); tsdbFSIterSeek(&pTsdbReadHandle->fileIter, fid);
tsdbUnLockFS(pFileHandle); tsdbUnLockFS(pFileHandle);
return getFirstFileDataBlock(pQueryHandle, exists); return getFirstFileDataBlock(pTsdbReadHandle, exists);
} else { } else {
// check if current file block is all consumed // check if current file block is all consumed
STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; STableBlockInfo* pBlockInfo = &pTsdbReadHandle->pDataBlockInfo[cur->slot];
STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo; STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo;
// current block is done, try next // current block is done, try next
if ((!cur->mixBlock) || cur->blockCompleted) { if ((!cur->mixBlock) || cur->blockCompleted) {
// all data blocks in current file has been checked already, try next file if exists // all data blocks in current file has been checked already, try next file if exists
} else { } else {
tsdbDebug("%p continue in current data block, index:%d, pos:%d, 0x%"PRIx64, pQueryHandle, cur->slot, cur->pos, tsdbDebug("%p continue in current data block, index:%d, pos:%d, 0x%"PRIx64, pTsdbReadHandle, cur->slot, cur->pos,
pQueryHandle->qId); pTsdbReadHandle->qId);
int32_t code = handleDataMergeIfNeeded(pQueryHandle, pBlockInfo->compBlock, pCheckInfo); int32_t code = handleDataMergeIfNeeded(pTsdbReadHandle, pBlockInfo->compBlock, pCheckInfo);
*exists = (pQueryHandle->realNumOfRows > 0); *exists = (pTsdbReadHandle->realNumOfRows > 0);
if (code != TSDB_CODE_SUCCESS || *exists) { if (code != TSDB_CODE_SUCCESS || *exists) {
return code; return code;
...@@ -2550,50 +2533,50 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists ...@@ -2550,50 +2533,50 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists
// current block is empty, try next block in file // current block is empty, try next block in file
// all data blocks in current file has been checked already, try next file if exists // all data blocks in current file has been checked already, try next file if exists
if (isEndFileDataBlock(cur, pQueryHandle->numOfBlocks, ASCENDING_TRAVERSE(pQueryHandle->order))) { if (isEndFileDataBlock(cur, pTsdbReadHandle->numOfBlocks, ASCENDING_TRAVERSE(pTsdbReadHandle->order))) {
return getFirstFileDataBlock(pQueryHandle, exists); return getFirstFileDataBlock(pTsdbReadHandle, exists);
} else { } else {
moveToNextDataBlockInCurrentFile(pQueryHandle); moveToNextDataBlockInCurrentFile(pTsdbReadHandle);
STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot]; STableBlockInfo* pNext = &pTsdbReadHandle->pDataBlockInfo[cur->slot];
return getDataBlockRv(pQueryHandle, pNext, exists); return getDataBlockRv(pTsdbReadHandle, pNext, exists);
} }
} }
} }
static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) { static bool doHasDataInBuffer(STsdbReadHandle* pTsdbReadHandle) {
size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo);
while (pQueryHandle->activeIndex < numOfTables) { while (pTsdbReadHandle->activeIndex < numOfTables) {
if (hasMoreDataInCache(pQueryHandle)) { if (hasMoreDataInCache(pTsdbReadHandle)) {
return true; return true;
} }
pQueryHandle->activeIndex += 1; pTsdbReadHandle->activeIndex += 1;
} }
// no data in memtable or imemtable, decrease the memory reference. // no data in memtable or imemtable, decrease the memory reference.
// TODO !! // TODO !!
// tsdbMayUnTakeMemSnapshot(pQueryHandle); // tsdbMayUnTakeMemSnapshot(pTsdbReadHandle);
return false; return false;
} }
//todo not unref yet, since it is not support multi-group interpolation query //todo not unref yet, since it is not support multi-group interpolation query
static UNUSED_FUNC void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { static UNUSED_FUNC void changeQueryHandleForInterpQuery(tsdbReadHandleT pHandle) {
// filter the queried time stamp in the first place // filter the queried time stamp in the first place
STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle;
// starts from the buffer in case of descending timestamp order check data blocks // starts from the buffer in case of descending timestamp order check data blocks
size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo);
int32_t i = 0; int32_t i = 0;
while(i < numOfTables) { while(i < numOfTables) {
STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i);
// the first qualified table for interpolation query // the first qualified table for interpolation query
if ((pQueryHandle->window.skey <= pCheckInfo->pTableObj->lastKey) && // if ((pTsdbReadHandle->window.skey <= pCheckInfo->pTableObj->lastKey) &&
(pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL)) { // (pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL)) {
break; // break;
} // }
i++; i++;
} }
...@@ -2603,35 +2586,34 @@ static UNUSED_FUNC void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle ...@@ -2603,35 +2586,34 @@ static UNUSED_FUNC void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle
return; return;
} }
STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i); STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i);
taosArrayClear(pQueryHandle->pTableCheckInfo); taosArrayClear(pTsdbReadHandle->pTableCheckInfo);
info.lastKey = pQueryHandle->window.skey; info.lastKey = pTsdbReadHandle->window.skey;
taosArrayPush(pQueryHandle->pTableCheckInfo, &info); taosArrayPush(pTsdbReadHandle->pTableCheckInfo, &info);
} }
static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win,
STsdbQueryHandle* pQueryHandle) { STsdbReadHandle* pTsdbReadHandle) {
int numOfRows = 0; int numOfRows = 0;
int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns); int32_t numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns);
STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config;
win->skey = TSKEY_INITIAL_VAL; win->skey = TSKEY_INITIAL_VAL;
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
STable* pTable = pCheckInfo->pTableObj;
int16_t rv = -1; int16_t rv = -1;
STSchema* pSchema = NULL; STSchema* pSchema = NULL;
do { do {
SMemRow row = getSMemRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update, NULL); SMemRow row = getSMemRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, NULL);
if (row == NULL) { if (row == NULL) {
break; break;
} }
TSKEY key = memRowKey(row); TSKEY key = memRowKey(row);
if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { if ((key > maxKey && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) {
tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey, tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pTsdbReadHandle, key, pTsdbReadHandle->window.skey,
pQueryHandle->window.ekey); pTsdbReadHandle->window.ekey);
break; break;
} }
...@@ -2642,10 +2624,10 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int ...@@ -2642,10 +2624,10 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
win->ekey = key; win->ekey = key;
if (rv != memRowVersion(row)) { if (rv != memRowVersion(row)) {
pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row)); pSchema = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, pCheckInfo->tableId, 0);
rv = memRowVersion(row); rv = memRowVersion(row);
} }
mergeTwoRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, NULL, numOfCols, pTable, pSchema, NULL, true); mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, numOfRows, row, NULL, numOfCols, pCheckInfo->tableId, pSchema, NULL, true);
if (++numOfRows >= maxRowsToRead) { if (++numOfRows >= maxRowsToRead) {
moveToNextRowInMem(pCheckInfo); moveToNextRowInMem(pCheckInfo);
...@@ -2657,24 +2639,24 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int ...@@ -2657,24 +2639,24 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
assert(numOfRows <= maxRowsToRead); assert(numOfRows <= maxRowsToRead);
// if the buffer is not full in case of descending order query, move the data in the front of the buffer // if the buffer is not full in case of descending order query, move the data in the front of the buffer
if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < maxRowsToRead) { if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && numOfRows < maxRowsToRead) {
int32_t emptySize = maxRowsToRead - numOfRows; int32_t emptySize = maxRowsToRead - numOfRows;
for(int32_t i = 0; i < numOfCols; ++i) { for(int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes);
} }
} }
int64_t elapsedTime = taosGetTimestampUs() - st; int64_t elapsedTime = taosGetTimestampUs() - st;
tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, 0x%"PRIx64, pQueryHandle, tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, 0x%"PRIx64, pTsdbReadHandle,
elapsedTime, numOfRows, numOfCols, pQueryHandle->qId); elapsedTime, numOfRows, numOfCols, pTsdbReadHandle->qId);
return numOfRows; return numOfRows;
} }
static int32_t getAllTableList(STable* pSuperTable, SArray* list) { static int32_t getAllTableList(STable* pSuperTable, SArray* list) {
SSkipListIterator* iter = tSkipListCreateIter(pSuperTable->pIndex); SSkipListIterator* iter = NULL;//tSkipListCreateIter(pSuperTable->pIndex);
while (tSkipListIterNext(iter)) { while (tSkipListIterNext(iter)) {
SSkipListNode* pNode = tSkipListIterGet(iter); SSkipListNode* pNode = tSkipListIterGet(iter);
...@@ -2693,58 +2675,61 @@ static void destroyHelper(void* param) { ...@@ -2693,58 +2675,61 @@ static void destroyHelper(void* param) {
return; return;
} }
tQueryInfo* pInfo = (tQueryInfo*)param; // tQueryInfo* pInfo = (tQueryInfo*)param;
if (pInfo->optr != TSDB_RELATION_IN) { // if (pInfo->optr != TSDB_RELATION_IN) {
tfree(pInfo->q); // tfree(pInfo->q);
} else { // } else {
taosHashCleanup((SHashObj *)(pInfo->q)); // taosHashCleanup((SHashObj *)(pInfo->q));
} // }
free(param); free(param);
} }
static bool loadBlockOfActiveTable(STsdbQueryHandle* pQueryHandle) { #define TSDB_PREV_ROW 0x1
if (pQueryHandle->checkFiles) { #define TSDB_NEXT_ROW 0x2
static bool loadBlockOfActiveTable(STsdbReadHandle* pTsdbReadHandle) {
if (pTsdbReadHandle->checkFiles) {
// check if the query range overlaps with the file data block // check if the query range overlaps with the file data block
bool exists = true; bool exists = true;
int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); int32_t code = getDataBlocksInFiles(pTsdbReadHandle, &exists);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
pQueryHandle->checkFiles = false; pTsdbReadHandle->checkFiles = false;
return false; return false;
} }
if (exists) { if (exists) {
tsdbRetrieveDataBlock((TsdbQueryHandleT*) pQueryHandle, NULL); tsdbRetrieveDataBlock((tsdbReadHandleT*) pTsdbReadHandle, NULL);
if (pQueryHandle->currentLoadExternalRows && pQueryHandle->window.skey == pQueryHandle->window.ekey) { if (pTsdbReadHandle->currentLoadExternalRows && pTsdbReadHandle->window.skey == pTsdbReadHandle->window.ekey) {
SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, 0); SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, 0);
assert(*(int64_t*)pColInfo->pData == pQueryHandle->window.skey); assert(*(int64_t*)pColInfo->pData == pTsdbReadHandle->window.skey);
} }
pQueryHandle->currentLoadExternalRows = false; // clear the flag, since the exact matched row is found. pTsdbReadHandle->currentLoadExternalRows = false; // clear the flag, since the exact matched row is found.
return exists; return exists;
} }
pQueryHandle->checkFiles = false; pTsdbReadHandle->checkFiles = false;
} }
if (hasMoreDataInCache(pQueryHandle)) { if (hasMoreDataInCache(pTsdbReadHandle)) {
pQueryHandle->currentLoadExternalRows = false; pTsdbReadHandle->currentLoadExternalRows = false;
return true; return true;
} }
// current result is empty // current result is empty
if (pQueryHandle->currentLoadExternalRows && pQueryHandle->window.skey == pQueryHandle->window.ekey && pQueryHandle->cur.rows == 0) { if (pTsdbReadHandle->currentLoadExternalRows && pTsdbReadHandle->window.skey == pTsdbReadHandle->window.ekey && pTsdbReadHandle->cur.rows == 0) {
SMemRef* pMemRef = pQueryHandle->pMemRef; // STsdbMemTable* pMemRef = pTsdbReadHandle->pMemTable;
doGetExternalRow(pQueryHandle, TSDB_PREV_ROW, pMemRef); // doGetExternalRow(pTsdbReadHandle, TSDB_PREV_ROW, pMemRef);
doGetExternalRow(pQueryHandle, TSDB_NEXT_ROW, pMemRef); // doGetExternalRow(pTsdbReadHandle, TSDB_NEXT_ROW, pMemRef);
bool result = tsdbGetExternalRow(pQueryHandle); bool result = tsdbGetExternalRow(pTsdbReadHandle);
pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); // pTsdbReadHandle->prev = doFreeColumnInfoData(pTsdbReadHandle->prev);
pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); // pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next);
pQueryHandle->currentLoadExternalRows = false; pTsdbReadHandle->currentLoadExternalRows = false;
return result; return result;
} }
...@@ -2752,26 +2737,26 @@ static bool loadBlockOfActiveTable(STsdbQueryHandle* pQueryHandle) { ...@@ -2752,26 +2737,26 @@ static bool loadBlockOfActiveTable(STsdbQueryHandle* pQueryHandle) {
return false; return false;
} }
static bool loadCachedLastRow(STsdbQueryHandle* pQueryHandle) { static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) {
// the last row is cached in buffer, return it directly. // the last row is cached in buffer, return it directly.
// here note that the pQueryHandle->window must be the TS_INITIALIZER // here note that the pTsdbReadHandle->window must be the TS_INITIALIZER
int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pTsdbReadHandle));
size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo);
assert(numOfTables > 0 && numOfCols > 0); assert(numOfTables > 0 && numOfCols > 0);
SQueryFilePos* cur = &pQueryHandle->cur; SQueryFilePos* cur = &pTsdbReadHandle->cur;
SMemRow pRow = NULL; SMemRow pRow = NULL;
TSKEY key = TSKEY_INITIAL_VAL; TSKEY key = TSKEY_INITIAL_VAL;
int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1:-1;
if (++pQueryHandle->activeIndex < numOfTables) { if (++pTsdbReadHandle->activeIndex < numOfTables) {
STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex);
int32_t ret = tsdbGetCachedLastRow(pCheckInfo->pTableObj, &pRow, &key); // int32_t ret = tsdbGetCachedLastRow(pCheckInfo->pTableObj, &pRow, &key);
if (ret != TSDB_CODE_SUCCESS) { // if (ret != TSDB_CODE_SUCCESS) {
return false; // return false;
} // }
mergeTwoRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, 0, pRow, NULL, numOfCols, pCheckInfo->pTableObj, NULL, NULL, true); mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, 0, pRow, NULL, numOfCols, pCheckInfo->pTableObj, NULL, NULL, true);
tfree(pRow); tfree(pRow);
// update the last key value // update the last key value
...@@ -2791,192 +2776,191 @@ static bool loadCachedLastRow(STsdbQueryHandle* pQueryHandle) { ...@@ -2791,192 +2776,191 @@ static bool loadCachedLastRow(STsdbQueryHandle* pQueryHandle) {
static bool loadCachedLast(STsdbQueryHandle* pQueryHandle) { //static bool loadCachedLast(STsdbReadHandle* pTsdbReadHandle) {
// the last row is cached in buffer, return it directly. // // the last row is cached in buffer, return it directly.
// here note that the pQueryHandle->window must be the TS_INITIALIZER // // here note that the pTsdbReadHandle->window must be the TS_INITIALIZER
int32_t tgNumOfCols = (int32_t)QH_GET_NUM_OF_COLS(pQueryHandle); // int32_t tgNumOfCols = (int32_t)QH_GET_NUM_OF_COLS(pTsdbReadHandle);
size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); // size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo);
int32_t numOfRows = 0; // int32_t numOfRows = 0;
assert(numOfTables > 0 && tgNumOfCols > 0); // assert(numOfTables > 0 && tgNumOfCols > 0);
SQueryFilePos* cur = &pQueryHandle->cur; // SQueryFilePos* cur = &pTsdbReadHandle->cur;
TSKEY priKey = TSKEY_INITIAL_VAL; // TSKEY priKey = TSKEY_INITIAL_VAL;
int32_t priIdx = -1; // int32_t priIdx = -1;
SColumnInfoData* pColInfo = NULL; // SColumnInfoData* pColInfo = NULL;
//
while (++pQueryHandle->activeIndex < numOfTables) { // while (++pTsdbReadHandle->activeIndex < numOfTables) {
STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); // STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex);
STable* pTable = pCheckInfo->pTableObj; // STable* pTable = pCheckInfo->pTableObj;
char* pData = NULL; // char* pData = NULL;
//
int32_t numOfCols = pTable->maxColNum; // int32_t numOfCols = pTable->maxColNum;
//
if (pTable->lastCols == NULL || pTable->maxColNum <= 0) { // if (pTable->lastCols == NULL || pTable->maxColNum <= 0) {
tsdbWarn("no last cached for table %s, uid:%" PRIu64 ",tid:%d", pTable->name->data, pTable->tableId.uid, pTable->tableId.tid); // tsdbWarn("no last cached for table %s, uid:%" PRIu64 ",tid:%d", pTable->name->data, pTable->uid, pTable->tableId);
continue; // continue;
} // }
//
int32_t i = 0, j = 0; // int32_t i = 0, j = 0;
while(i < tgNumOfCols && j < numOfCols) { // while(i < tgNumOfCols && j < numOfCols) {
pColInfo = taosArrayGet(pQueryHandle->pColumns, i); // pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
if (pTable->lastCols[j].colId < pColInfo->info.colId) { // if (pTable->lastCols[j].colId < pColInfo->info.colId) {
j++; // j++;
continue; // continue;
} else if (pTable->lastCols[j].colId > pColInfo->info.colId) { // } else if (pTable->lastCols[j].colId > pColInfo->info.colId) {
i++; // i++;
continue; // continue;
} // }
//
pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; // pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;
//
if (pTable->lastCols[j].bytes > 0) { // if (pTable->lastCols[j].bytes > 0) {
void* value = pTable->lastCols[j].pData; // void* value = pTable->lastCols[j].pData;
switch (pColInfo->info.type) { // switch (pColInfo->info.type) {
case TSDB_DATA_TYPE_BINARY: // case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR: // case TSDB_DATA_TYPE_NCHAR:
memcpy(pData, value, varDataTLen(value)); // memcpy(pData, value, varDataTLen(value));
break; // break;
case TSDB_DATA_TYPE_NULL: // case TSDB_DATA_TYPE_NULL:
case TSDB_DATA_TYPE_BOOL: // case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT: // case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_UTINYINT: // case TSDB_DATA_TYPE_UTINYINT:
*(uint8_t *)pData = *(uint8_t *)value; // *(uint8_t *)pData = *(uint8_t *)value;
break; // break;
case TSDB_DATA_TYPE_SMALLINT: // case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_USMALLINT: // case TSDB_DATA_TYPE_USMALLINT:
*(uint16_t *)pData = *(uint16_t *)value; // *(uint16_t *)pData = *(uint16_t *)value;
break; // break;
case TSDB_DATA_TYPE_INT: // case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_UINT: // case TSDB_DATA_TYPE_UINT:
*(uint32_t *)pData = *(uint32_t *)value; // *(uint32_t *)pData = *(uint32_t *)value;
break; // break;
case TSDB_DATA_TYPE_BIGINT: // case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_UBIGINT: // case TSDB_DATA_TYPE_UBIGINT:
*(uint64_t *)pData = *(uint64_t *)value; // *(uint64_t *)pData = *(uint64_t *)value;
break; // break;
case TSDB_DATA_TYPE_FLOAT: // case TSDB_DATA_TYPE_FLOAT:
SET_FLOAT_PTR(pData, value); // SET_FLOAT_PTR(pData, value);
break; // break;
case TSDB_DATA_TYPE_DOUBLE: // case TSDB_DATA_TYPE_DOUBLE:
SET_DOUBLE_PTR(pData, value); // SET_DOUBLE_PTR(pData, value);
break; // break;
case TSDB_DATA_TYPE_TIMESTAMP: // case TSDB_DATA_TYPE_TIMESTAMP:
if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { // if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
priKey = tdGetKey(*(TKEY *)value); // priKey = tdGetKey(*(TKEY *)value);
priIdx = i; // priIdx = i;
//
i++; // i++;
j++; // j++;
continue; // continue;
} else { // } else {
*(TSKEY *)pData = *(TSKEY *)value; // *(TSKEY *)pData = *(TSKEY *)value;
} // }
break; // break;
default: // default:
memcpy(pData, value, pColInfo->info.bytes); // memcpy(pData, value, pColInfo->info.bytes);
} // }
//
for (int32_t n = 0; n < tgNumOfCols; ++n) { // for (int32_t n = 0; n < tgNumOfCols; ++n) {
if (n == i) { // if (n == i) {
continue; // continue;
} // }
//
pColInfo = taosArrayGet(pQueryHandle->pColumns, n); // pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, n);
pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; // pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;;
//
if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { // if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
*(TSKEY *)pData = pTable->lastCols[j].ts; //// *(TSKEY *)pData = pTable->lastCols[j].ts;
continue; // continue;
} // }
//
if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { // if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
setVardataNull(pData, pColInfo->info.type); // setVardataNull(pData, pColInfo->info.type);
} else { // } else {
setNull(pData, pColInfo->info.type, pColInfo->info.bytes); // setNull(pData, pColInfo->info.type, pColInfo->info.bytes);
} // }
} // }
//
numOfRows++; // numOfRows++;
assert(numOfRows < pQueryHandle->outputCapacity); // assert(numOfRows < pTsdbReadHandle->outputCapacity);
} // }
//
i++; // i++;
j++; // j++;
} // }
//
// leave the real ts column as the last row, because last function only (not stable) use the last row as res // // leave the real ts column as the last row, because last function only (not stable) use the last row as res
if (priKey != TSKEY_INITIAL_VAL) { // if (priKey != TSKEY_INITIAL_VAL) {
pColInfo = taosArrayGet(pQueryHandle->pColumns, priIdx); // pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, priIdx);
pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; // pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;
//
*(TSKEY *)pData = priKey; // *(TSKEY *)pData = priKey;
//
for (int32_t n = 0; n < tgNumOfCols; ++n) { // for (int32_t n = 0; n < tgNumOfCols; ++n) {
if (n == priIdx) { // if (n == priIdx) {
continue; // continue;
} // }
//
pColInfo = taosArrayGet(pQueryHandle->pColumns, n); // pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, n);
pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; // pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;;
//
assert (pColInfo->info.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX); // assert (pColInfo->info.colId != PRIMARYKEY_TIMESTAMP_COL_ID);
//
if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { // if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
setVardataNull(pData, pColInfo->info.type); // setVardataNull(pData, pColInfo->info.type);
} else { // } else {
setNull(pData, pColInfo->info.type, pColInfo->info.bytes); // setNull(pData, pColInfo->info.type, pColInfo->info.bytes);
} // }
} // }
//
numOfRows++; // numOfRows++;
} // }
//
if (numOfRows > 0) { // if (numOfRows > 0) {
cur->rows = numOfRows; // cur->rows = numOfRows;
cur->mixBlock = true; // cur->mixBlock = true;
//
return true; // return true;
} // }
} // }
//
return false; // return false;
} //}
static bool loadDataBlockFromTableSeq(STsdbReadHandle* pTsdbReadHandle) {
static bool loadDataBlockFromTableSeq(STsdbQueryHandle* pQueryHandle) { size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo);
size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
assert(numOfTables > 0); assert(numOfTables > 0);
int64_t stime = taosGetTimestampUs(); int64_t stime = taosGetTimestampUs();
while(pQueryHandle->activeIndex < numOfTables) { while(pTsdbReadHandle->activeIndex < numOfTables) {
if (loadBlockOfActiveTable(pQueryHandle)) { if (loadBlockOfActiveTable(pTsdbReadHandle)) {
return true; return true;
} }
STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex);
pCheckInfo->numOfBlocks = 0; pCheckInfo->numOfBlocks = 0;
pQueryHandle->activeIndex += 1; pTsdbReadHandle->activeIndex += 1;
pQueryHandle->locateStart = false; pTsdbReadHandle->locateStart = false;
pQueryHandle->checkFiles = true; pTsdbReadHandle->checkFiles = true;
pQueryHandle->cur.rows = 0; pTsdbReadHandle->cur.rows = 0;
pQueryHandle->currentLoadExternalRows = pQueryHandle->loadExternalRow; pTsdbReadHandle->currentLoadExternalRows = pTsdbReadHandle->loadExternalRow;
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
int64_t elapsedTime = taosGetTimestampUs() - stime; int64_t elapsedTime = taosGetTimestampUs() - stime;
pQueryHandle->cost.checkForNextTime += elapsedTime; pTsdbReadHandle->cost.checkForNextTime += elapsedTime;
} }
return false; return false;
} }
// handle data in cache situation // handle data in cache situation
bool tsdbNextDataBlock(TsdbQueryHandleT pHandle) { bool tsdbNextDataBlock(tsdbReadHandleT pHandle) {
STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle;
if (emptyQueryTimewindow(pQueryHandle)) { if (emptyQueryTimewindow(pTsdbReadHandle)) {
tsdbDebug("%p query window not overlaps with the data set, no result returned, 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); tsdbDebug("%p query window not overlaps with the data set, no result returned, 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId);
return false; return false;
} }
...@@ -2984,177 +2968,177 @@ bool tsdbNextDataBlock(TsdbQueryHandleT pHandle) { ...@@ -2984,177 +2968,177 @@ bool tsdbNextDataBlock(TsdbQueryHandleT pHandle) {
int64_t elapsedTime = stime; int64_t elapsedTime = stime;
// TODO refactor: remove "type" // TODO refactor: remove "type"
if (pQueryHandle->type == TSDB_QUERY_TYPE_LAST) { if (pTsdbReadHandle->type == TSDB_QUERY_TYPE_LAST) {
if (pQueryHandle->cachelastrow == TSDB_CACHED_TYPE_LASTROW) { if (pTsdbReadHandle->cachelastrow == TSDB_CACHED_TYPE_LASTROW) {
return loadCachedLastRow(pQueryHandle); // return loadCachedLastRow(pTsdbReadHandle);
} else if (pQueryHandle->cachelastrow == TSDB_CACHED_TYPE_LAST) { } else if (pTsdbReadHandle->cachelastrow == TSDB_CACHED_TYPE_LAST) {
return loadCachedLast(pQueryHandle); // return loadCachedLast(pTsdbReadHandle);
} }
} }
if (pQueryHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { if (pTsdbReadHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) {
return loadDataBlockFromTableSeq(pQueryHandle); return loadDataBlockFromTableSeq(pTsdbReadHandle);
} else { // loadType == RR and Offset Order } else { // loadType == RR and Offset Order
if (pQueryHandle->checkFiles) { if (pTsdbReadHandle->checkFiles) {
// check if the query range overlaps with the file data block // check if the query range overlaps with the file data block
bool exists = true; bool exists = true;
int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); int32_t code = getDataBlocksInFiles(pTsdbReadHandle, &exists);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
pQueryHandle->activeIndex = 0; pTsdbReadHandle->activeIndex = 0;
pQueryHandle->checkFiles = false; pTsdbReadHandle->checkFiles = false;
return false; return false;
} }
if (exists) { if (exists) {
pQueryHandle->cost.checkForNextTime += (taosGetTimestampUs() - stime); pTsdbReadHandle->cost.checkForNextTime += (taosGetTimestampUs() - stime);
return exists; return exists;
} }
pQueryHandle->activeIndex = 0; pTsdbReadHandle->activeIndex = 0;
pQueryHandle->checkFiles = false; pTsdbReadHandle->checkFiles = false;
} }
// TODO: opt by consider the scan order // TODO: opt by consider the scan order
bool ret = doHasDataInBuffer(pQueryHandle); bool ret = doHasDataInBuffer(pTsdbReadHandle);
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
elapsedTime = taosGetTimestampUs() - stime; elapsedTime = taosGetTimestampUs() - stime;
pQueryHandle->cost.checkForNextTime += elapsedTime; pTsdbReadHandle->cost.checkForNextTime += elapsedTime;
return ret; return ret;
} }
} }
static int32_t doGetExternalRow(STsdbQueryHandle* pQueryHandle, int16_t type, SMemRef* pMemRef) { //static int32_t doGetExternalRow(STsdbReadHandle* pTsdbReadHandle, int16_t type, STsdbMemTable* pMemRef) {
STsdbQueryHandle* pSecQueryHandle = NULL; // STsdbReadHandle* pSecQueryHandle = NULL;
//
if (type == TSDB_PREV_ROW && pQueryHandle->prev) { // if (type == TSDB_PREV_ROW && pTsdbReadHandle->prev) {
return TSDB_CODE_SUCCESS; // return TSDB_CODE_SUCCESS;
} // }
//
if (type == TSDB_NEXT_ROW && pQueryHandle->next) { // if (type == TSDB_NEXT_ROW && pTsdbReadHandle->next) {
return TSDB_CODE_SUCCESS; // return TSDB_CODE_SUCCESS;
} // }
//
// prepare the structure // // prepare the structure
int32_t numOfCols = (int32_t) QH_GET_NUM_OF_COLS(pQueryHandle); // int32_t numOfCols = (int32_t) QH_GET_NUM_OF_COLS(pTsdbReadHandle);
//
if (type == TSDB_PREV_ROW) { // if (type == TSDB_PREV_ROW) {
pQueryHandle->prev = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); // pTsdbReadHandle->prev = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
if (pQueryHandle->prev == NULL) { // if (pTsdbReadHandle->prev == NULL) {
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; // terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
goto out_of_memory; // goto out_of_memory;
} // }
} else { // } else {
pQueryHandle->next = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); // pTsdbReadHandle->next = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
if (pQueryHandle->next == NULL) { // if (pTsdbReadHandle->next == NULL) {
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; // terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
goto out_of_memory; // goto out_of_memory;
} // }
} // }
//
SArray* row = (type == TSDB_PREV_ROW)? pQueryHandle->prev : pQueryHandle->next; // SArray* row = (type == TSDB_PREV_ROW)? pTsdbReadHandle->prev : pTsdbReadHandle->next;
//
for (int32_t i = 0; i < numOfCols; ++i) { // for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); // SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pColumns, i);
//
SColumnInfoData colInfo = {{0}, 0}; // SColumnInfoData colInfo = {{0}, 0};
colInfo.info = pCol->info; // colInfo.info = pCol->info;
colInfo.pData = calloc(1, pCol->info.bytes); // colInfo.pData = calloc(1, pCol->info.bytes);
if (colInfo.pData == NULL) { // if (colInfo.pData == NULL) {
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; // terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
goto out_of_memory; // goto out_of_memory;
} // }
//
taosArrayPush(row, &colInfo); // taosArrayPush(row, &colInfo);
} // }
//
// load the previous row // // load the previous row
STsdbQueryCond cond = {.numOfCols = numOfCols, .loadExternalRows = false, .type = BLOCK_LOAD_OFFSET_SEQ_ORDER}; // STsdbQueryCond cond = {.numOfCols = numOfCols, .loadExternalRows = false, .type = BLOCK_LOAD_OFFSET_SEQ_ORDER};
if (type == TSDB_PREV_ROW) { // if (type == TSDB_PREV_ROW) {
cond.order = TSDB_ORDER_DESC; // cond.order = TSDB_ORDER_DESC;
cond.twindow = (STimeWindow){pQueryHandle->window.skey, INT64_MIN}; // cond.twindow = (STimeWindow){pTsdbReadHandle->window.skey, INT64_MIN};
} else { // } else {
cond.order = TSDB_ORDER_ASC; // cond.order = TSDB_ORDER_ASC;
cond.twindow = (STimeWindow){pQueryHandle->window.skey, INT64_MAX}; // cond.twindow = (STimeWindow){pTsdbReadHandle->window.skey, INT64_MAX};
} // }
//
cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); // cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo));
if (cond.colList == NULL) { // if (cond.colList == NULL) {
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; // terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
goto out_of_memory; // goto out_of_memory;
} // }
//
for (int32_t i = 0; i < cond.numOfCols; ++i) { // for (int32_t i = 0; i < cond.numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, i); // SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, i);
memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo)); // memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo));
} // }
//
pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qId, pMemRef); // pSecQueryHandle = tsdbQueryTablesImpl(pTsdbReadHandle->pTsdb, &cond, pTsdbReadHandle->qId, pMemRef);
tfree(cond.colList); // tfree(cond.colList);
//
// current table, only one table // // current table, only one table
STableCheckInfo* pCurrent = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); // STableCheckInfo* pCurrent = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex);
//
SArray* psTable = NULL; // SArray* psTable = NULL;
pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pCurrent, pSecQueryHandle->window.skey, &psTable); // pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pCurrent, pSecQueryHandle->window.skey, &psTable);
if (pSecQueryHandle->pTableCheckInfo == NULL) { // if (pSecQueryHandle->pTableCheckInfo == NULL) {
taosArrayDestroy(psTable); // taosArrayDestroy(psTable);
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; // terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
goto out_of_memory; // goto out_of_memory;
} // }
//
//
tsdbMayTakeMemSnapshot(pSecQueryHandle, psTable); // tsdbMayTakeMemSnapshot(pSecQueryHandle, psTable);
if (!tsdbNextDataBlock((void*)pSecQueryHandle)) { // if (!tsdbNextDataBlock((void*)pSecQueryHandle)) {
// no result in current query, free the corresponding result rows structure // // no result in current query, free the corresponding result rows structure
if (type == TSDB_PREV_ROW) { // if (type == TSDB_PREV_ROW) {
pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); // pTsdbReadHandle->prev = doFreeColumnInfoData(pTsdbReadHandle->prev);
} else { // } else {
pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); // pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next);
} // }
//
goto out_of_memory; // goto out_of_memory;
} // }
//
SDataBlockInfo blockInfo = {{0}, 0}; // SDataBlockInfo blockInfo = {{0}, 0};
tsdbRetrieveDataBlockInfo((void*)pSecQueryHandle, &blockInfo); // tsdbRetrieveDataBlockInfo((void*)pSecQueryHandle, &blockInfo);
tsdbRetrieveDataBlock((void*)pSecQueryHandle, pSecQueryHandle->defaultLoadColumn); // tsdbRetrieveDataBlock((void*)pSecQueryHandle, pSecQueryHandle->defaultLoadColumn);
//
row = (type == TSDB_PREV_ROW)? pQueryHandle->prev:pQueryHandle->next; // row = (type == TSDB_PREV_ROW)? pTsdbReadHandle->prev:pTsdbReadHandle->next;
int32_t pos = (type == TSDB_PREV_ROW)?pSecQueryHandle->cur.rows - 1:0; // int32_t pos = (type == TSDB_PREV_ROW)?pSecQueryHandle->cur.rows - 1:0;
//
for (int32_t i = 0; i < numOfCols; ++i) { // for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(row, i); // SColumnInfoData* pCol = taosArrayGet(row, i);
SColumnInfoData* s = taosArrayGet(pSecQueryHandle->pColumns, i); // SColumnInfoData* s = taosArrayGet(pSecQueryHandle->pColumns, i);
memcpy((char*)pCol->pData, (char*)s->pData + s->info.bytes * pos, pCol->info.bytes); // memcpy((char*)pCol->pData, (char*)s->pData + s->info.bytes * pos, pCol->info.bytes);
} // }
//
out_of_memory: //out_of_memory:
tsdbCleanupQueryHandle(pSecQueryHandle); // tsdbCleanupQueryHandle(pSecQueryHandle);
return terrno; // return terrno;
} //}
bool tsdbGetExternalRow(TsdbQueryHandleT pHandle) { bool tsdbGetExternalRow(tsdbReadHandleT pHandle) {
STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle;
SQueryFilePos* cur = &pQueryHandle->cur; SQueryFilePos* cur = &pTsdbReadHandle->cur;
cur->fid = INT32_MIN; cur->fid = INT32_MIN;
cur->mixBlock = true; cur->mixBlock = true;
if (pQueryHandle->prev == NULL || pQueryHandle->next == NULL) { if (pTsdbReadHandle->prev == NULL || pTsdbReadHandle->next == NULL) {
cur->rows = 0; cur->rows = 0;
return false; return false;
} }
int32_t numOfCols = (int32_t) QH_GET_NUM_OF_COLS(pQueryHandle); int32_t numOfCols = (int32_t) QH_GET_NUM_OF_COLS(pTsdbReadHandle);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, i); SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, i);
SColumnInfoData* first = taosArrayGet(pQueryHandle->prev, i); SColumnInfoData* first = taosArrayGet(pTsdbReadHandle->prev, i);
memcpy(pColInfoData->pData, first->pData, pColInfoData->info.bytes); memcpy(pColInfoData->pData, first->pData, pColInfoData->info.bytes);
SColumnInfoData* sec = taosArrayGet(pQueryHandle->next, i); SColumnInfoData* sec = taosArrayGet(pTsdbReadHandle->next, i);
memcpy(((char*)pColInfoData->pData) + pColInfoData->info.bytes, sec->pData, pColInfoData->info.bytes); memcpy(((char*)pColInfoData->pData) + pColInfoData->info.bytes, sec->pData, pColInfoData->info.bytes);
if (i == 0 && pColInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP) { if (i == 0 && pColInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP) {
...@@ -3171,76 +3155,75 @@ bool tsdbGetExternalRow(TsdbQueryHandleT pHandle) { ...@@ -3171,76 +3155,75 @@ bool tsdbGetExternalRow(TsdbQueryHandleT pHandle) {
* if lastRow == NULL, return TSDB_CODE_TDB_NO_CACHE_LAST_ROW * if lastRow == NULL, return TSDB_CODE_TDB_NO_CACHE_LAST_ROW
* else set pRes and return TSDB_CODE_SUCCESS and save lastKey * else set pRes and return TSDB_CODE_SUCCESS and save lastKey
*/ */
int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey) { //int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey) {
int32_t code = TSDB_CODE_SUCCESS; // int32_t code = TSDB_CODE_SUCCESS;
//
TSDB_RLOCK_TABLE(pTable); // TSDB_RLOCK_TABLE(pTable);
//
if (!pTable->lastRow) { // if (!pTable->lastRow) {
code = TSDB_CODE_TDB_NO_CACHE_LAST_ROW; // code = TSDB_CODE_TDB_NO_CACHE_LAST_ROW;
goto out; // goto out;
} // }
//
if (pRes) { // if (pRes) {
*pRes = tdMemRowDup(pTable->lastRow); // *pRes = tdMemRowDup(pTable->lastRow);
if (*pRes == NULL) { // if (*pRes == NULL) {
code = TSDB_CODE_TDB_OUT_OF_MEMORY; // code = TSDB_CODE_TDB_OUT_OF_MEMORY;
} // }
} // }
//
out: //out:
TSDB_RUNLOCK_TABLE(pTable); // TSDB_RUNLOCK_TABLE(pTable);
return code; // return code;
} //}
bool isTsdbCacheLastRow(TsdbQueryHandleT* pQueryHandle) { bool isTsdbCacheLastRow(tsdbReadHandleT* pTsdbReadHandle) {
return ((STsdbQueryHandle *)pQueryHandle)->cachelastrow > TSDB_CACHED_TYPE_NONE; return ((STsdbReadHandle *)pTsdbReadHandle)->cachelastrow > TSDB_CACHED_TYPE_NONE;
} }
int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList) { int32_t checkForCachedLastRow(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo *groupList) {
assert(pQueryHandle != NULL && groupList != NULL); assert(pTsdbReadHandle != NULL && groupList != NULL);
TSKEY key = TSKEY_INITIAL_VAL; // TSKEY key = TSKEY_INITIAL_VAL;
//
SArray* group = taosArrayGetP(groupList->pGroupList, 0); // SArray* group = taosArrayGetP(groupList->pGroupList, 0);
assert(group != NULL); // assert(group != NULL);
//
STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(group, 0); // STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(group, 0);
//
int32_t code = 0; // int32_t code = 0;
//
if (((STable*)pInfo->pTable)->lastRow) { // if (((STable*)pInfo->pTable)->lastRow) {
code = tsdbGetCachedLastRow(pInfo->pTable, NULL, &key); // code = tsdbGetCachedLastRow(pInfo->pTable, NULL, &key);
if (code != TSDB_CODE_SUCCESS) { // if (code != TSDB_CODE_SUCCESS) {
pQueryHandle->cachelastrow = TSDB_CACHED_TYPE_NONE; // pTsdbReadHandle->cachelastrow = TSDB_CACHED_TYPE_NONE;
} else { // } else {
pQueryHandle->cachelastrow = TSDB_CACHED_TYPE_LASTROW; // pTsdbReadHandle->cachelastrow = TSDB_CACHED_TYPE_LASTROW;
} // }
} // }
//
// update the tsdb query time range // // update the tsdb query time range
if (pQueryHandle->cachelastrow != TSDB_CACHED_TYPE_NONE) { // if (pTsdbReadHandle->cachelastrow != TSDB_CACHED_TYPE_NONE) {
pQueryHandle->window = TSWINDOW_INITIALIZER; // pTsdbReadHandle->window = TSWINDOW_INITIALIZER;
pQueryHandle->checkFiles = false; // pTsdbReadHandle->checkFiles = false;
pQueryHandle->activeIndex = -1; // start from -1 // pTsdbReadHandle->activeIndex = -1; // start from -1
} // }
return code; return TSDB_CODE_SUCCESS;
} }
int32_t checkForCachedLast(STsdbQueryHandle* pQueryHandle) { int32_t checkForCachedLast(STsdbReadHandle* pTsdbReadHandle) {
assert(pQueryHandle != NULL); assert(pTsdbReadHandle != NULL);
int32_t code = 0; int32_t code = 0;
// if (pTsdbReadHandle->pTsdb && atomic_load_8(&pTsdbReadHandle->pTsdb->hasCachedLastColumn)){
if (pQueryHandle->pTsdb && atomic_load_8(&pQueryHandle->pTsdb->hasCachedLastColumn)){ // pTsdbReadHandle->cachelastrow = TSDB_CACHED_TYPE_LAST;
pQueryHandle->cachelastrow = TSDB_CACHED_TYPE_LAST; // }
}
// update the tsdb query time range // update the tsdb query time range
if (pQueryHandle->cachelastrow) { if (pTsdbReadHandle->cachelastrow) {
pQueryHandle->checkFiles = false; pTsdbReadHandle->checkFiles = false;
pQueryHandle->activeIndex = -1; // start from -1 pTsdbReadHandle->activeIndex = -1; // start from -1
} }
return code; return code;
...@@ -3266,7 +3249,7 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { ...@@ -3266,7 +3249,7 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) {
STableKeyInfo* pInfo = (STableKeyInfo*) taosArrayGet(pGroup, i); STableKeyInfo* pInfo = (STableKeyInfo*) taosArrayGet(pGroup, i);
// if the lastKey equals to INT64_MIN, there is no data in this table // if the lastKey equals to INT64_MIN, there is no data in this table
TSKEY lastKey = ((STable*)(pInfo->pTable))->lastKey; TSKEY lastKey = 0;//((STable*)(pInfo->pTable))->lastKey;
if (key < lastKey) { if (key < lastKey) {
key = lastKey; key = lastKey;
...@@ -3290,7 +3273,7 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { ...@@ -3290,7 +3273,7 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) {
// keyInfo.pTable may be NULL here. // keyInfo.pTable may be NULL here.
if (pInfo->pTable != keyInfo.pTable) { if (pInfo->pTable != keyInfo.pTable) {
tsdbUnRefTable(pInfo->pTable); // tsdbUnRefTable(pInfo->pTable);
} }
} }
...@@ -3322,23 +3305,23 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { ...@@ -3322,23 +3305,23 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) {
return window; return window;
} }
void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* pDataBlockInfo) { void tsdbRetrieveDataBlockInfo(tsdbReadHandleT* pTsdbReadHandle, SDataBlockInfo* pDataBlockInfo) {
STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle; STsdbReadHandle* pHandle = (STsdbReadHandle*)pTsdbReadHandle;
SQueryFilePos* cur = &pHandle->cur; SQueryFilePos* cur = &pHandle->cur;
STable* pTable = NULL;
uint64_t uid = 0;
// there are data in file // there are data in file
if (pHandle->cur.fid != INT32_MIN) { if (pHandle->cur.fid != INT32_MIN) {
STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[cur->slot]; STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[cur->slot];
pTable = pBlockInfo->pTableCheckInfo->pTableObj; uid = pBlockInfo->pTableCheckInfo->tableId;
} else { } else {
STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex);
pTable = pCheckInfo->pTableObj; uid = pCheckInfo->tableId;
} }
pDataBlockInfo->uid = pTable->tableId.uid; pDataBlockInfo->uid = uid;
pDataBlockInfo->tid = pTable->tableId.tid; pDataBlockInfo->rows = cur->rows;
pDataBlockInfo->rows = cur->rows;
pDataBlockInfo->window = cur->win; pDataBlockInfo->window = cur->win;
pDataBlockInfo->numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pHandle)); pDataBlockInfo->numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pHandle));
} }
...@@ -3346,8 +3329,8 @@ void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* p ...@@ -3346,8 +3329,8 @@ void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* p
/* /*
* return null for mixed data block, if not a complete file data block, the statistics value will always return NULL * return null for mixed data block, if not a complete file data block, the statistics value will always return NULL
*/ */
int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataStatis** pBlockStatis) { int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReadHandleT* pTsdbReadHandle, SDataStatis** pBlockStatis) {
STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; STsdbReadHandle* pHandle = (STsdbReadHandle*) pTsdbReadHandle;
SQueryFilePos* c = &pHandle->cur; SQueryFilePos* c = &pHandle->cur;
if (c->mixBlock) { if (c->mixBlock) {
...@@ -3381,7 +3364,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta ...@@ -3381,7 +3364,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta
// always load the first primary timestamp column data // always load the first primary timestamp column data
SDataStatis* pPrimaryColStatis = &pHandle->statis[0]; SDataStatis* pPrimaryColStatis = &pHandle->statis[0];
assert(pPrimaryColStatis->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX); assert(pPrimaryColStatis->colId == PRIMARYKEY_TIMESTAMP_COL_ID);
pPrimaryColStatis->numOfNull = 0; pPrimaryColStatis->numOfNull = 0;
pPrimaryColStatis->min = pBlockInfo->compBlock->keyFirst; pPrimaryColStatis->min = pBlockInfo->compBlock->keyFirst;
...@@ -3401,12 +3384,12 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta ...@@ -3401,12 +3384,12 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { SArray* tsdbRetrieveDataBlock(tsdbReadHandleT* pTsdbReadHandle, SArray* pIdList) {
/** /**
* In the following two cases, the data has been loaded to SColumnInfoData. * In the following two cases, the data has been loaded to SColumnInfoData.
* 1. data is from cache, 2. data block is not completed qualified to query time range * 1. data is from cache, 2. data block is not completed qualified to query time range
*/ */
STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle; STsdbReadHandle* pHandle = (STsdbReadHandle*)pTsdbReadHandle;
if (pHandle->cur.fid == INT32_MIN) { if (pHandle->cur.fid == INT32_MIN) {
return pHandle->pColumns; return pHandle->pColumns;
...@@ -3417,14 +3400,14 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { ...@@ -3417,14 +3400,14 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) {
if (pHandle->cur.mixBlock) { if (pHandle->cur.mixBlock) {
return pHandle->pColumns; return pHandle->pColumns;
} else { } else {
SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlockInfo->compBlock); SDataBlockInfo binfo = {0};/*GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlockInfo->compBlock);*/
assert(pHandle->realNumOfRows <= binfo.rows); assert(pHandle->realNumOfRows <= binfo.rows);
// data block has been loaded, todo extract method // data block has been loaded, todo extract method
SDataBlockLoadInfo* pBlockLoadInfo = &pHandle->dataBlockLoadInfo; SDataBlockLoadInfo* pBlockLoadInfo = &pHandle->dataBlockLoadInfo;
if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fid == pHandle->cur.fid && if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fid == pHandle->cur.fid &&
pBlockLoadInfo->tid == pCheckInfo->pTableObj->tableId.tid) { pBlockLoadInfo->uid == pCheckInfo->pTableObj->tid) {
return pHandle->pColumns; return pHandle->pColumns;
} else { // only load the file block } else { // only load the file block
SBlock* pBlock = pBlockInfo->compBlock; SBlock* pBlock = pBlockInfo->compBlock;
...@@ -3451,7 +3434,7 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { ...@@ -3451,7 +3434,7 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) {
} }
} }
} }
#if 0
void filterPrepare(void* expr, void* param) { void filterPrepare(void* expr, void* param) {
tExprNode* pExpr = (tExprNode*)expr; tExprNode* pExpr = (tExprNode*)expr;
if (pExpr->_node.info != NULL) { if (pExpr->_node.info != NULL) {
...@@ -3550,11 +3533,12 @@ static int32_t tableGroupComparFn(const void *p1, const void *p2, const void *pa ...@@ -3550,11 +3533,12 @@ static int32_t tableGroupComparFn(const void *p1, const void *p2, const void *pa
return 0; return 0;
} }
#endif
static int tsdbCheckInfoCompar(const void* key1, const void* key2) { static int tsdbCheckInfoCompar(const void* key1, const void* key2) {
if (((STableCheckInfo*)key1)->tableId.tid < ((STableCheckInfo*)key2)->tableId.tid) { if (((STableCheckInfo*)key1)->tableId < ((STableCheckInfo*)key2)->tableId) {
return -1; return -1;
} else if (((STableCheckInfo*)key1)->tableId.tid > ((STableCheckInfo*)key2)->tableId.tid) { } else if (((STableCheckInfo*)key1)->tableId > ((STableCheckInfo*)key2)->tableId) {
return 1; return 1;
} else { } else {
ASSERT(false); ASSERT(false);
...@@ -3570,7 +3554,6 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable ...@@ -3570,7 +3554,6 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable
STableKeyInfo info = {.pTable = pTable, .lastKey = skey}; STableKeyInfo info = {.pTable = pTable, .lastKey = skey};
taosArrayPush(g, &info); taosArrayPush(g, &info);
tsdbRefTable(pTable);
for (int32_t i = 1; i < numOfTables; ++i) { for (int32_t i = 1; i < numOfTables; ++i) {
STable** prev = taosArrayGet(pTableList, i - 1); STable** prev = taosArrayGet(pTableList, i - 1);
...@@ -3579,8 +3562,7 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable ...@@ -3579,8 +3562,7 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable
int32_t ret = compareFn(prev, p, pSupp); int32_t ret = compareFn(prev, p, pSupp);
assert(ret == 0 || ret == -1); assert(ret == 0 || ret == -1);
tsdbRefTable(*p); // assert((*p)->type == TSDB_CHILD_TABLE);
assert((*p)->type == TSDB_CHILD_TABLE);
if (ret == 0) { if (ret == 0) {
STableKeyInfo info1 = {.pTable = *p, .lastKey = skey}; STableKeyInfo info1 = {.pTable = *p, .lastKey = skey};
...@@ -3597,6 +3579,7 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable ...@@ -3597,6 +3579,7 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable
taosArrayPush(pGroups, &g); taosArrayPush(pGroups, &g);
} }
#if 0
SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols, TSKEY skey) { SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols, TSKEY skey) {
assert(pTableList != NULL); assert(pTableList != NULL);
SArray* pTableGroup = taosArrayInit(1, POINTER_BYTES); SArray* pTableGroup = taosArrayInit(1, POINTER_BYTES);
...@@ -3616,7 +3599,6 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC ...@@ -3616,7 +3599,6 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC
for(int32_t i = 0; i < size; ++i) { for(int32_t i = 0; i < size; ++i) {
STableKeyInfo *pKeyInfo = taosArrayGet(pTableList, i); STableKeyInfo *pKeyInfo = taosArrayGet(pTableList, i);
tsdbRefTable(pKeyInfo->pTable);
STableKeyInfo info = {.pTable = pKeyInfo->pTable, .lastKey = skey}; STableKeyInfo info = {.pTable = pKeyInfo->pTable, .lastKey = skey};
taosArrayPush(sa, &info); taosArrayPush(sa, &info);
...@@ -3737,7 +3719,7 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) ...@@ -3737,7 +3719,7 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr)
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len,
int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo, int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo,
SColIndex* pColIndex, int32_t numOfCols) { SColIndex* pColIndex, int32_t numOfCols) {
if (tsdbRLockRepoMeta(tsdb) < 0) goto _error; if (tsdbRLockRepoMeta(tsdb) < 0) goto _error;
...@@ -3752,7 +3734,7 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, cons ...@@ -3752,7 +3734,7 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, cons
} }
if (pTable->type != TSDB_SUPER_TABLE) { if (pTable->type != TSDB_SUPER_TABLE) {
tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", tid:%d, name:%s", tsdb, uid, pTable->tableId.tid, tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", tid:%d, name:%s", tsdb, uid, pTable->tableId,
pTable->name->data); pTable->name->data);
terrno = TSDB_CODE_COM_OPS_NOT_SUPPORT; //basically, this error is caused by invalid sql issued by client terrno = TSDB_CODE_COM_OPS_NOT_SUPPORT; //basically, this error is caused by invalid sql issued by client
...@@ -3821,8 +3803,8 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, cons ...@@ -3821,8 +3803,8 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, cons
pGroupInfo->numOfTables = (uint32_t)taosArrayGetSize(res); pGroupInfo->numOfTables = (uint32_t)taosArrayGetSize(res);
pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, skey); pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, skey);
tsdbDebug("%p stable tid:%d, uid:%"PRIu64" query, numOfTables:%u, belong to %" PRIzu " groups", tsdb, pTable->tableId.tid, tsdbDebug("%p stable tid:%d, uid:%"PRIu64" query, numOfTables:%u, belong to %" PRIzu " groups", tsdb, pTable->tableId,
pTable->tableId.uid, pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList)); pTable->uid, pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList));
taosArrayDestroy(res); taosArrayDestroy(res);
...@@ -3833,7 +3815,7 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, cons ...@@ -3833,7 +3815,7 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, cons
return terrno; return terrno;
} }
int32_t tsdbGetOneTableGroup(STsdbRepo* tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) { int32_t tsdbGetOneTableGroup(STsdb* tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) {
if (tsdbRLockRepoMeta(tsdb) < 0) goto _error; if (tsdbRLockRepoMeta(tsdb) < 0) goto _error;
STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid);
...@@ -3844,7 +3826,6 @@ int32_t tsdbGetOneTableGroup(STsdbRepo* tsdb, uint64_t uid, TSKEY startKey, STab ...@@ -3844,7 +3826,6 @@ int32_t tsdbGetOneTableGroup(STsdbRepo* tsdb, uint64_t uid, TSKEY startKey, STab
} }
assert(pTable->type == TSDB_CHILD_TABLE || pTable->type == TSDB_NORMAL_TABLE || pTable->type == TSDB_STREAM_TABLE); assert(pTable->type == TSDB_CHILD_TABLE || pTable->type == TSDB_NORMAL_TABLE || pTable->type == TSDB_STREAM_TABLE);
tsdbRefTable(pTable);
if (tsdbUnlockRepoMeta(tsdb) < 0) goto _error; if (tsdbUnlockRepoMeta(tsdb) < 0) goto _error;
pGroupInfo->numOfTables = 1; pGroupInfo->numOfTables = 1;
...@@ -3862,7 +3843,7 @@ int32_t tsdbGetOneTableGroup(STsdbRepo* tsdb, uint64_t uid, TSKEY startKey, STab ...@@ -3862,7 +3843,7 @@ int32_t tsdbGetOneTableGroup(STsdbRepo* tsdb, uint64_t uid, TSKEY startKey, STab
return terrno; return terrno;
} }
int32_t tsdbGetTableGroupFromIdList(STsdbRepo* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo) { int32_t tsdbGetTableGroupFromIdList(STsdb* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo) {
if (tsdbRLockRepoMeta(tsdb) < 0) { if (tsdbRLockRepoMeta(tsdb) < 0) {
return terrno; return terrno;
} }
...@@ -3889,8 +3870,6 @@ int32_t tsdbGetTableGroupFromIdList(STsdbRepo* tsdb, SArray* pTableIdList, STabl ...@@ -3889,8 +3870,6 @@ int32_t tsdbGetTableGroupFromIdList(STsdbRepo* tsdb, SArray* pTableIdList, STabl
return terrno; return terrno;
} }
tsdbRefTable(pTable);
STableKeyInfo info = {.pTable = pTable, .lastKey = id->key}; STableKeyInfo info = {.pTable = pTable, .lastKey = id->key};
taosArrayPush(group, &info); taosArrayPush(group, &info);
} }
...@@ -3938,42 +3917,42 @@ static void* destroyTableCheckInfo(SArray* pTableCheckInfo) { ...@@ -3938,42 +3917,42 @@ static void* destroyTableCheckInfo(SArray* pTableCheckInfo) {
return NULL; return NULL;
} }
void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { void tsdbCleanupQueryHandle(tsdbReadHandleT queryHandle) {
STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*)queryHandle; STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)queryHandle;
if (pQueryHandle == NULL) { if (pTsdbReadHandle == NULL) {
return; return;
} }
pQueryHandle->pColumns = doFreeColumnInfoData(pQueryHandle->pColumns); pTsdbReadHandle->pColumns = doFreeColumnInfoData(pTsdbReadHandle->pColumns);
taosArrayDestroy(pQueryHandle->defaultLoadColumn); taosArrayDestroy(pTsdbReadHandle->defaultLoadColumn);
tfree(pQueryHandle->pDataBlockInfo); tfree(pTsdbReadHandle->pDataBlockInfo);
tfree(pQueryHandle->statis); tfree(pTsdbReadHandle->statis);
if (!emptyQueryTimewindow(pQueryHandle)) { if (!emptyQueryTimewindow(pTsdbReadHandle)) {
tsdbMayUnTakeMemSnapshot(pQueryHandle); tsdbMayUnTakeMemSnapshot(pTsdbReadHandle);
} else { } else {
assert(pQueryHandle->pTableCheckInfo == NULL); assert(pTsdbReadHandle->pTableCheckInfo == NULL);
} }
if (pQueryHandle->pTableCheckInfo != NULL) { if (pTsdbReadHandle->pTableCheckInfo != NULL) {
pQueryHandle->pTableCheckInfo = destroyTableCheckInfo(pQueryHandle->pTableCheckInfo); pTsdbReadHandle->pTableCheckInfo = destroyTableCheckInfo(pTsdbReadHandle->pTableCheckInfo);
} }
tsdbDestroyReadH(&pQueryHandle->rhelper); tsdbDestroyReadH(&pTsdbReadHandle->rhelper);
tdFreeDataCols(pQueryHandle->pDataCols); tdFreeDataCols(pTsdbReadHandle->pDataCols);
pQueryHandle->pDataCols = NULL; pTsdbReadHandle->pDataCols = NULL;
pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); pTsdbReadHandle->prev = doFreeColumnInfoData(pTsdbReadHandle->prev);
pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next);
SIOCostSummary* pCost = &pQueryHandle->cost; SIOCostSummary* pCost = &pTsdbReadHandle->cost;
tsdbDebug("%p :io-cost summary: head-file read cnt:%"PRIu64", head-file time:%"PRIu64" us, statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, 0x%"PRIx64, tsdbDebug("%p :io-cost summary: head-file read cnt:%"PRIu64", head-file time:%"PRIu64" us, statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, 0x%"PRIx64,
pQueryHandle, pCost->headFileLoad, pCost->headFileLoadTime, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qId); pTsdbReadHandle, pCost->headFileLoad, pCost->headFileLoadTime, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pTsdbReadHandle->qId);
tfree(pQueryHandle); tfree(pTsdbReadHandle);
} }
void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) { void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) {
...@@ -4257,3 +4236,4 @@ void getTableListfromSkipList(tExprNode *pExpr, SSkipList *pSkipList, SArray *re ...@@ -4257,3 +4236,4 @@ void getTableListfromSkipList(tExprNode *pExpr, SSkipList *pSkipList, SArray *re
//apply the hierarchical filter expression to every node in skiplist to find the qualified nodes //apply the hierarchical filter expression to every node in skiplist to find the qualified nodes
applyFilterToSkipListNode(pSkipList, pExpr, result, param); applyFilterToSkipListNode(pSkipList, pExpr, result, param);
} }
#endif
\ No newline at end of file
source/libs/executor/CMakeLists.txt
浏览文件 @ 369b5559
aux_source_directory(src EXECUTOR_SRC) aux_source_directory(src EXECUTOR_SRC)
add_library(executor ${EXECUTOR_SRC}) #add_library(executor ${EXECUTOR_SRC})
#target_link_libraries(
# executor
# PRIVATE os util common function parser planner qcom tsdb
#)
add_library(executor STATIC ${EXECUTOR_SRC})
#set_target_properties(executor PROPERTIES
# IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/libexecutor.a"
# INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/libs/executor"
# )
target_link_libraries(executor
PRIVATE os util common function parser planner qcom tsdb
)
target_include_directories( target_include_directories(
executor executor
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/executor" PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/executor"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
) )
target_link_libraries( #if(${BUILD_TEST})
executor ADD_SUBDIRECTORY(test)
PRIVATE os util common function parser planner qcom #endif(${BUILD_TEST})
) \ No newline at end of file
\ No newline at end of file
source/libs/executor/inc/executil.h
浏览文件 @ 369b5559
...@@ -38,7 +38,7 @@ ...@@ -38,7 +38,7 @@
#define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t)) #define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t))
#define GET_RES_EXT_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t) + POINTER_BYTES) #define GET_RES_EXT_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t) + POINTER_BYTES)
#define GET_QID(_r) (((SQInfo*)((_r)->qinfo))->qId) #define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.queryId)
#define curTimeWindowIndex(_winres) ((_winres)->curIndex) #define curTimeWindowIndex(_winres) ((_winres)->curIndex)
...@@ -157,6 +157,6 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); ...@@ -157,6 +157,6 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo);
int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, struct STaskRuntimeEnv *pRuntimeEnv, int32_t* offset); int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, struct STaskRuntimeEnv *pRuntimeEnv, int32_t* offset);
int32_t initUdfInfo(struct SUdfInfo* pUdfInfo); //int32_t initUdfInfo(struct SUdfInfo* pUdfInfo);
#endif // TDENGINE_QUERYUTIL_H #endif // TDENGINE_QUERYUTIL_H
source/libs/executor/inc/executorInt.h
浏览文件 @ 369b5559
...@@ -20,6 +20,7 @@ ...@@ -20,6 +20,7 @@
extern "C" { extern "C" {
#endif #endif
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
......
source/libs/executor/inc/executorimpl.h
浏览文件 @ 369b5559
...@@ -20,24 +20,19 @@ ...@@ -20,24 +20,19 @@
#include "ttszip.h" #include "ttszip.h"
#include "tvariant.h" #include "tvariant.h"
#include "thash.h" #include "dataSinkMgt.h"
#include "executil.h" #include "executil.h"
#include "planner.h"
#include "taosdef.h" #include "taosdef.h"
#include "tarray.h" #include "tarray.h"
#include "tfilter.h" #include "tfilter.h"
#include "thash.h"
#include "tlockfree.h" #include "tlockfree.h"
#include "tpagedfile.h" #include "tpagedfile.h"
#include "planner.h" #include "executor.h"
struct SColumnFilterElem; struct SColumnFilterElem;
typedef struct {
uint32_t numOfTables;
SArray *pGroupList;
SHashObj *map; // speedup acquire the tableQueryInfo by table uid
} STableGroupInfo;
typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order);
#define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED) #define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED)
...@@ -51,19 +46,19 @@ typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int ...@@ -51,19 +46,19 @@ typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int
#define NEEDTO_COMPRESS_QUERY(size) ((size) > tsCompressColData? 1 : 0) #define NEEDTO_COMPRESS_QUERY(size) ((size) > tsCompressColData? 1 : 0)
enum { enum {
// when query starts to execute, this status will set // when this task starts to execute, this status will set
QUERY_NOT_COMPLETED = 0x1u, TASK_NOT_COMPLETED = 0x1u,
/* query is over /* Task is over
* 1. this status is used in one row result query process, e.g., count/sum/first/last/ avg...etc. * 1. this status is used in one row result query process, e.g., count/sum/first/last/ avg...etc.
* 2. when all data within queried time window, it is also denoted as query_completed * 2. when all data within queried time window, it is also denoted as query_completed
*/ */
QUERY_COMPLETED = 0x2u, TASK_COMPLETED = 0x2u,
/* when the result is not completed return to client, this status will be /* when the result is not completed return to client, this status will be
* usually used in case of interval query with interpolation option * usually used in case of interval query with interpolation option
*/ */
QUERY_OVER = 0x4u, TASK_OVER = 0x4u,
}; };
typedef struct SResultRowCell { typedef struct SResultRowCell {
...@@ -129,6 +124,7 @@ typedef struct { ...@@ -129,6 +124,7 @@ typedef struct {
} SOperatorProfResult; } SOperatorProfResult;
typedef struct STaskCostInfo { typedef struct STaskCostInfo {
int64_t created;
int64_t start; int64_t start;
int64_t end; int64_t end;
...@@ -246,13 +242,14 @@ typedef struct STaskIdInfo { ...@@ -246,13 +242,14 @@ typedef struct STaskIdInfo {
uint64_t taskId; // this is a subplan id uint64_t taskId; // this is a subplan id
} STaskIdInfo; } STaskIdInfo;
typedef struct STaskInfo { typedef struct SExecTaskInfo {
STaskIdInfo id; STaskIdInfo id;
char *content; char *content;
uint32_t status; uint32_t status;
STimeWindow window; STimeWindow window;
STaskCostInfo cost; STaskCostInfo cost;
int64_t owner; // if it is in execution int64_t owner; // if it is in execution
int32_t code;
STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray<STableQueryInfo*> structure STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray<STableQueryInfo*> structure
pthread_mutex_t lock; // used to synchronize the rsp/query threads pthread_mutex_t lock; // used to synchronize the rsp/query threads
...@@ -260,8 +257,10 @@ typedef struct STaskInfo { ...@@ -260,8 +257,10 @@ typedef struct STaskInfo {
// int32_t dataReady; // denote if query result is ready or not // int32_t dataReady; // denote if query result is ready or not
// void* rspContext; // response context // void* rspContext; // response context
char *sql; // query sql string char *sql; // query sql string
jmp_buf env; jmp_buf env; //
} STaskInfo; DataSinkHandle dsHandle;
struct SOperatorInfo *pRoot;
} SExecTaskInfo;
typedef struct STaskRuntimeEnv { typedef struct STaskRuntimeEnv {
jmp_buf env; jmp_buf env;
...@@ -269,7 +268,7 @@ typedef struct STaskRuntimeEnv { ...@@ -269,7 +268,7 @@ typedef struct STaskRuntimeEnv {
uint32_t status; // query status uint32_t status; // query status
void* qinfo; void* qinfo;
uint8_t scanFlag; // denotes reversed scan of data or not uint8_t scanFlag; // denotes reversed scan of data or not
void* pQueryHandle; void* pTsdbReadHandle;
int32_t prevGroupId; // previous executed group id int32_t prevGroupId; // previous executed group id
bool enableGroupData; bool enableGroupData;
...@@ -314,8 +313,8 @@ typedef struct SOperatorInfo { ...@@ -314,8 +313,8 @@ typedef struct SOperatorInfo {
char *name; // name, used to show the query execution plan char *name; // name, used to show the query execution plan
void *info; // extension attribution void *info; // extension attribution
SExprInfo *pExpr; SExprInfo *pExpr;
STaskRuntimeEnv *pRuntimeEnv; STaskRuntimeEnv *pRuntimeEnv; // todo remove it
STaskInfo *pTaskInfo; SExecTaskInfo *pTaskInfo;
struct SOperatorInfo **pDownstream; // downstram pointer list struct SOperatorInfo **pDownstream; // downstram pointer list
int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator
...@@ -376,7 +375,7 @@ typedef struct STaskParam { ...@@ -376,7 +375,7 @@ typedef struct STaskParam {
} STaskParam; } STaskParam;
typedef struct STableScanInfo { typedef struct STableScanInfo {
void *pQueryHandle; void *pTsdbReadHandle;
int32_t numOfBlocks; int32_t numOfBlocks;
int32_t numOfSkipped; int32_t numOfSkipped;
int32_t numOfBlockStatis; int32_t numOfBlockStatis;
...@@ -544,7 +543,7 @@ typedef struct SOrderOperatorInfo { ...@@ -544,7 +543,7 @@ typedef struct SOrderOperatorInfo {
void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream); void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream);
SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime); SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime);
SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime); SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv);
SOperatorInfo* createAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
...@@ -572,11 +571,11 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI ...@@ -572,11 +571,11 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI
SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pUpstream, int32_t numOfUpstream, SSchema* pSchema, int32_t numOfOutput); SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pUpstream, int32_t numOfUpstream, SSchema* pSchema, int32_t numOfOutput);
SOperatorInfo* createOrderOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrder* pOrderVal); SOperatorInfo* createOrderOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrder* pOrderVal);
SSDataBlock* doGlobalAggregate(void* param, bool* newgroup); //SSDataBlock* doGlobalAggregate(void* param, bool* newgroup);
SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup); //SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup);
SSDataBlock* doSLimit(void* param, bool* newgroup); //SSDataBlock* doSLimit(void* param, bool* newgroup);
int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId); //int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId);
void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock); void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock);
bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p); bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p);
void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p); void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p);
...@@ -617,14 +616,14 @@ STableQueryInfo* createTmpTableQueryInfo(STimeWindow win); ...@@ -617,14 +616,14 @@ STableQueryInfo* createTmpTableQueryInfo(STimeWindow win);
int32_t buildArithmeticExprFromMsg(SExprInfo *pArithExprInfo, void *pQueryMsg); int32_t buildArithmeticExprFromMsg(SExprInfo *pArithExprInfo, void *pQueryMsg);
bool isQueryKilled(SQInfo *pQInfo); bool isTaskKilled(SExecTaskInfo *pTaskInfo);
int32_t checkForQueryBuf(size_t numOfTables); int32_t checkForQueryBuf(size_t numOfTables);
bool checkNeedToCompressQueryCol(SQInfo *pQInfo); bool checkNeedToCompressQueryCol(SQInfo *pQInfo);
bool doBuildResCheck(SQInfo* pQInfo); bool doBuildResCheck(SQInfo* pQInfo);
void setQueryStatus(STaskRuntimeEnv *pRuntimeEnv, int8_t status); void setQueryStatus(STaskRuntimeEnv *pRuntimeEnv, int8_t status);
bool onlyQueryTags(STaskAttr* pQueryAttr); bool onlyQueryTags(STaskAttr* pQueryAttr);
void destroyUdfInfo(struct SUdfInfo* pUdfInfo); //void destroyUdfInfo(struct SUdfInfo* pUdfInfo);
bool isValidQInfo(void *param); bool isValidQInfo(void *param);
...@@ -644,5 +643,7 @@ void freeQueryAttr(STaskAttr *pQuery); ...@@ -644,5 +643,7 @@ void freeQueryAttr(STaskAttr *pQuery);
int32_t getMaximumIdleDurationSec(); int32_t getMaximumIdleDurationSec();
void doInvokeUdf(struct SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type); void doInvokeUdf(struct SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type);
void setTaskStatus(SExecTaskInfo *pTaskInfo, int8_t status);
int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle);
#endif // TDENGINE_EXECUTORIMPL_H #endif // TDENGINE_EXECUTORIMPL_H
source/libs/executor/src/dataDispatcher.c
浏览文件 @ 369b5559
...@@ -109,10 +109,14 @@ static void toDataCacheEntry(const SDataDispatchHandle* pHandle, const SInputDat ...@@ -109,10 +109,14 @@ static void toDataCacheEntry(const SDataDispatchHandle* pHandle, const SInputDat
SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData; SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData;
pEntry->compressed = (int8_t)needCompress(pInput->pData, &(pHandle->schema)); pEntry->compressed = (int8_t)needCompress(pInput->pData, &(pHandle->schema));
pEntry->numOfRows = pInput->pData->info.rows; pEntry->numOfRows = pInput->pData->info.rows;
pEntry->dataLen = 0;
pBuf->useSize = DATA_META_LENGTH(pInput->pTableRetrieveTsMap); pBuf->useSize = DATA_META_LENGTH(pInput->pTableRetrieveTsMap);
copyData(pInput, &pHandle->schema, pEntry->data, pEntry->compressed, &pEntry->dataLen); copyData(pInput, &pHandle->schema, pEntry->data, pEntry->compressed, &pEntry->dataLen);
pBuf->useSize += (pEntry->compressed ? pEntry->dataLen : pHandle->schema.resultRowSize * pInput->pData->info.rows); if (0 == pEntry->compressed) {
pEntry->dataLen = pHandle->schema.resultRowSize * pInput->pData->info.rows;
}
pBuf->useSize += pEntry->dataLen;
// todo completed // todo completed
} }
...@@ -213,6 +217,7 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSink* pDataS ...@@ -213,6 +217,7 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSink* pDataS
return TSDB_CODE_QRY_OUT_OF_MEMORY; return TSDB_CODE_QRY_OUT_OF_MEMORY;
} }
dispatcher->sink.fPut = putDataBlock; dispatcher->sink.fPut = putDataBlock;
dispatcher->sink.fEndPut = endPut;
dispatcher->sink.fGetLen = getDataLength; dispatcher->sink.fGetLen = getDataLength;
dispatcher->sink.fGetData = getDataBlock; dispatcher->sink.fGetData = getDataBlock;
dispatcher->sink.fDestroy = destroyDataSinker; dispatcher->sink.fDestroy = destroyDataSinker;
......
source/libs/executor/src/dataSinkMgt.c
浏览文件 @ 369b5559
...@@ -13,6 +13,7 @@ ...@@ -13,6 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "tarray.h"
#include "dataSinkMgt.h" #include "dataSinkMgt.h"
#include "dataSinkInt.h" #include "dataSinkInt.h"
#include "planner.h" #include "planner.h"
......
source/libs/executor/src/executil.c
浏览文件 @ 369b5559
...@@ -547,7 +547,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv ...@@ -547,7 +547,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv
pTableQueryInfoList = malloc(POINTER_BYTES * size); pTableQueryInfoList = malloc(POINTER_BYTES * size);
if (pTableQueryInfoList == NULL || posList == NULL || pGroupResInfo->pRows == NULL || pGroupResInfo->pRows == NULL) { if (pTableQueryInfoList == NULL || posList == NULL || pGroupResInfo->pRows == NULL || pGroupResInfo->pRows == NULL) {
// qError("QInfo:%"PRIu64" failed alloc memory", GET_QID(pRuntimeEnv)); // qError("QInfo:%"PRIu64" failed alloc memory", GET_TASKID(pRuntimeEnv));
code = TSDB_CODE_QRY_OUT_OF_MEMORY; code = TSDB_CODE_QRY_OUT_OF_MEMORY;
goto _end; goto _end;
} }
...@@ -619,7 +619,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv ...@@ -619,7 +619,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv
int64_t endt = taosGetTimestampMs(); int64_t endt = taosGetTimestampMs();
// qDebug("QInfo:%"PRIx64" result merge completed for group:%d, elapsed time:%" PRId64 " ms", GET_QID(pRuntimeEnv), // qDebug("QInfo:%"PRIx64" result merge completed for group:%d, elapsed time:%" PRId64 " ms", GET_TASKID(pRuntimeEnv),
// pGroupResInfo->currentGroup, endt - startt); // pGroupResInfo->currentGroup, endt - startt);
_end: _end:
...@@ -641,13 +641,13 @@ int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, STaskRuntimeEnv* pRun ...@@ -641,13 +641,13 @@ int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, STaskRuntimeEnv* pRun
break; break;
} }
// qDebug("QInfo:%"PRIu64" no result in group %d, continue", GET_QID(pRuntimeEnv), pGroupResInfo->currentGroup); // qDebug("QInfo:%"PRIu64" no result in group %d, continue", GET_TASKID(pRuntimeEnv), pGroupResInfo->currentGroup);
cleanupGroupResInfo(pGroupResInfo); cleanupGroupResInfo(pGroupResInfo);
incNextGroup(pGroupResInfo); incNextGroup(pGroupResInfo);
} }
// int64_t elapsedTime = taosGetTimestampUs() - st; // int64_t elapsedTime = taosGetTimestampUs() - st;
// qDebug("QInfo:%"PRIu64" merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", GET_QID(pRuntimeEnv), // qDebug("QInfo:%"PRIu64" merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", GET_TASKID(pRuntimeEnv),
// pGroupResInfo->currentGroup, pGroupResInfo->totalGroup, elapsedTime); // pGroupResInfo->currentGroup, pGroupResInfo->totalGroup, elapsedTime);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
......
source/libs/executor/src/executorMain.c
浏览文件 @ 369b5559
...@@ -14,10 +14,12 @@ ...@@ -14,10 +14,12 @@
*/ */
#include "os.h" #include "os.h"
#include "tarray.h"
#include "dataSinkMgt.h"
#include "exception.h"
#include "tcache.h" #include "tcache.h"
#include "tglobal.h" #include "tglobal.h"
#include "tmsg.h" #include "tmsg.h"
#include "exception.h"
#include "thash.h" #include "thash.h"
#include "executorimpl.h" #include "executorimpl.h"
...@@ -66,152 +68,24 @@ void freeParam(STaskParam *param) { ...@@ -66,152 +68,24 @@ void freeParam(STaskParam *param) {
tfree(param->prevResult); tfree(param->prevResult);
} }
// todo parse json to get the operator tree. int32_t qCreateExecTask(void* tsdb, int32_t vgId, SSubplan* pSubplan, qTaskInfo_t* pTaskInfo) {
assert(tsdb != NULL && pSubplan != NULL);
int32_t qCreateTask(void* tsdb, int32_t vgId, void* pQueryMsg, qTaskInfo_t* pTaskInfo, uint64_t taskId) { SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo;
assert(pQueryMsg != NULL && tsdb != NULL);
int32_t code = TSDB_CODE_SUCCESS; int32_t code = doCreateExecTaskInfo(pSubplan, pTask, tsdb);
#if 0
STaskParam param = {0};
code = convertQueryMsg(pQueryMsg, &param);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _over; goto _error;
}
if (pQueryMsg->numOfTables <= 0) {
qError("Invalid number of tables to query, numOfTables:%d", pQueryMsg->numOfTables);
code = TSDB_CODE_QRY_INVALID_MSG;
goto _over;
}
if (param.pTableIdList == NULL || taosArrayGetSize(param.pTableIdList) == 0) {
qError("qmsg:%p, SQueryTableReq wrong format", pQueryMsg);
code = TSDB_CODE_QRY_INVALID_MSG;
goto _over;
}
SQueriedTableInfo info = { .numOfTags = pQueryMsg->numOfTags, .numOfCols = pQueryMsg->numOfCols, .colList = pQueryMsg->tableCols};
if ((code = createQueryFunc(&info, pQueryMsg->numOfOutput, &param.pExprs, param.pExpr, param.pTagColumnInfo,
pQueryMsg->queryType, pQueryMsg, param.pUdfInfo)) != TSDB_CODE_SUCCESS) {
goto _over;
}
if (param.pSecExpr != NULL) {
if ((code = createIndirectQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->secondStageOutput, &param.pSecExprs, param.pSecExpr, param.pExprs, param.pUdfInfo)) != TSDB_CODE_SUCCESS) {
goto _over;
}
}
if (param.colCond != NULL) {
if ((code = createQueryFilter(param.colCond, pQueryMsg->colCondLen, &param.pFilters)) != TSDB_CODE_SUCCESS) {
goto _over;
}
}
param.pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, param.pGroupColIndex, &code);
if ((param.pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) {
goto _over;
}
bool isSTableQuery = false;
STableGroupInfo tableGroupInfo = {0};
int64_t st = taosGetTimestampUs();
if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_TABLE_QUERY)) {
STableIdInfo *id = taosArrayGet(param.pTableIdList, 0);
qDebug("qmsg:%p query normal table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid);
if ((code = tsdbGetOneTableGroup(tsdb, id->uid, pQueryMsg->window.skey, &tableGroupInfo)) != TSDB_CODE_SUCCESS) {
goto _over;
}
} else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_STABLE_QUERY)) {
isSTableQuery = true;
// also note there's possibility that only one table in the super table
if (!TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY)) {
STableIdInfo *id = taosArrayGet(param.pTableIdList, 0);
// group by normal column, do not pass the group by condition to tsdb to group table into different group
int32_t numOfGroupByCols = pQueryMsg->numOfGroupCols;
if (pQueryMsg->numOfGroupCols == 1 && !TSDB_COL_IS_TAG(param.pGroupColIndex->flag)) {
numOfGroupByCols = 0;
}
qDebug("qmsg:%p query stable, uid:%"PRIu64", tid:%d", pQueryMsg, id->uid, id->tid);
code = tsdbQuerySTableByTagCond(tsdb, id->uid, pQueryMsg->window.skey, param.tagCond, pQueryMsg->tagCondLen,
pQueryMsg->tagNameRelType, param.tbnameCond, &tableGroupInfo, param.pGroupColIndex, numOfGroupByCols);
if (code != TSDB_CODE_SUCCESS) {
qError("qmsg:%p failed to query stable, reason: %s", pQueryMsg, tstrerror(code));
goto _over;
}
} else {
code = tsdbGetTableGroupFromIdList(tsdb, param.pTableIdList, &tableGroupInfo);
if (code != TSDB_CODE_SUCCESS) {
goto _over;
}
qDebug("qmsg:%p query on %u tables in one group from client", pQueryMsg, tableGroupInfo.numOfTables);
}
int64_t el = taosGetTimestampUs() - st;
qDebug("qmsg:%p tag filter completed, numOfTables:%u, elapsed time:%"PRId64"us", pQueryMsg, tableGroupInfo.numOfTables, el);
} else {
assert(0);
} }
code = checkForQueryBuf(tableGroupInfo.numOfTables); SDataSinkMgtCfg cfg = {.maxDataBlockNum = 1000, .maxDataBlockNumPerQuery = 100};
if (code != TSDB_CODE_SUCCESS) { // not enough query buffer, abort code = dsDataSinkMgtInit(&cfg);
goto _over;
}
assert(pQueryMsg->stableQuery == isSTableQuery);
(*pTaskInfo) = createQInfoImpl(pQueryMsg, param.pGroupbyExpr, param.pExprs, param.pSecExprs, &tableGroupInfo,
param.pTagColumnInfo, param.pFilters, vgId, param.sql, qId, param.pUdfInfo);
param.sql = NULL;
param.pExprs = NULL;
param.pSecExprs = NULL;
param.pGroupbyExpr = NULL;
param.pTagColumnInfo = NULL;
param.pFilters = NULL;
if ((*pTaskInfo) == NULL) {
code = TSDB_CODE_QRY_OUT_OF_MEMORY;
goto _over;
}
param.pUdfInfo = NULL;
code = initQInfo(&pQueryMsg->tsBuf, tsdb, NULL, *pTaskInfo, &param, (char*)pQueryMsg, pQueryMsg->prevResultLen, NULL);
_over:
if (param.pGroupbyExpr != NULL) {
taosArrayDestroy(param.pGroupbyExpr->columnInfo);
}
tfree(param.colCond);
destroyUdfInfo(param.pUdfInfo);
taosArrayDestroy(param.pTableIdList);
param.pTableIdList = NULL;
freeParam(&param);
for (int32_t i = 0; i < pQueryMsg->numOfCols; i++) {
SColumnInfo* column = pQueryMsg->tableCols + i;
freeColumnFilterInfo(column->flist.filterInfo, column->flist.numOfFilters);
}
filterFreeInfo(param.pFilters);
//pTaskInfo already freed in initQInfo, but *pTaskInfo may not pointer to null;
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
*pTaskInfo = NULL; goto _error;
} }
#endif
code = dsCreateDataSinker(pSubplan->pDataSink, &(*pTask)->dsHandle);
_error:
// if failed to add ref for all tables in this query, abort current query // if failed to add ref for all tables in this query, abort current query
return code; return code;
} }
...@@ -250,7 +124,7 @@ int waitMoment(SQInfo* pQInfo){ ...@@ -250,7 +124,7 @@ int waitMoment(SQInfo* pQInfo){
while(used_ms < ms) { while(used_ms < ms) {
taosMsleep(1000); taosMsleep(1000);
used_ms += 1000; used_ms += 1000;
if(isQueryKilled(pQInfo)){ if(isTaskKilled(pQInfo)){
printf("test check query is canceled, sleep break.%s\n", pQInfo->sql); printf("test check query is canceled, sleep break.%s\n", pQInfo->sql);
break; break;
} }
...@@ -261,68 +135,80 @@ int waitMoment(SQInfo* pQInfo){ ...@@ -261,68 +135,80 @@ int waitMoment(SQInfo* pQInfo){
} }
#endif #endif
bool qExecTask(qTaskInfo_t qinfo, uint64_t *qId) { int32_t qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle) {
SQInfo *pQInfo = (SQInfo *)qinfo; SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
assert(pQInfo && pQInfo->signature == pQInfo); int64_t threadId = taosGetSelfPthreadId();
int64_t threadId = taosGetSelfPthreadId();
int64_t curOwner = 0; int64_t curOwner = 0;
if ((curOwner = atomic_val_compare_exchange_64(&pQInfo->owner, 0, threadId)) != 0) { if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) {
qError("QInfo:0x%"PRIx64"-%p qhandle is now executed by thread:%p", pQInfo->qId, pQInfo, (void*) curOwner); qError("QInfo:0x%" PRIx64 "-%p qhandle is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo,
pQInfo->code = TSDB_CODE_QRY_IN_EXEC; (void*)curOwner);
return false; pTaskInfo->code = TSDB_CODE_QRY_IN_EXEC;
return pTaskInfo->code;
} }
*qId = pQInfo->qId; if (pTaskInfo->cost.start == 0) {
if(pQInfo->startExecTs == 0) pTaskInfo->cost.start = taosGetTimestampMs();
pQInfo->startExecTs = taosGetTimestampMs();
if (isQueryKilled(pQInfo)) {
qDebug("QInfo:0x%"PRIx64" it is already killed, abort", pQInfo->qId);
return doBuildResCheck(pQInfo);
} }
STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; if (isTaskKilled(pTaskInfo)) {
if (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 0) { qDebug("QInfo:0x%" PRIx64 " it is already killed, abort", GET_TASKID(pTaskInfo));
qDebug("QInfo:0x%"PRIx64" no table exists for query, abort", pQInfo->qId); return pTaskInfo->code;
// setTaskStatus(pRuntimeEnv, QUERY_COMPLETED);
return doBuildResCheck(pQInfo);
} }
// STaskRuntimeEnv* pRuntimeEnv = &pTaskInfo->runtimeEnv;
// if (pTaskInfo->tableqinfoGroupInfo.numOfTables == 0) {
// qDebug("QInfo:0x%"PRIx64" no table exists for query, abort", GET_TASKID(pTaskInfo));
// setTaskStatus(pTaskInfo, TASK_COMPLETED);
// return doBuildResCheck(pTaskInfo);
// }
// error occurs, record the error code and return to client // error occurs, record the error code and return to client
int32_t ret = setjmp(pQInfo->runtimeEnv.env); int32_t ret = setjmp(pTaskInfo->env);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
publishQueryAbortEvent(pQInfo, ret); publishQueryAbortEvent(pTaskInfo, ret);
pQInfo->code = ret; pTaskInfo->code = ret;
qDebug("QInfo:0x%"PRIx64" query abort due to error/cancel occurs, code:%s", pQInfo->qId, tstrerror(pQInfo->code)); qDebug("QInfo:0x%" PRIx64 " query abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code));
return doBuildResCheck(pQInfo); return pTaskInfo->code;
} }
qDebug("QInfo:0x%"PRIx64" query task is launched", pQInfo->qId); qDebug("QInfo:0x%" PRIx64 " query task is launched", GET_TASKID(pTaskInfo));
bool newgroup = false; bool newgroup = false;
publishOperatorProfEvent(pRuntimeEnv->proot, QUERY_PROF_BEFORE_OPERATOR_EXEC); publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_BEFORE_OPERATOR_EXEC);
int64_t st = 0;
int64_t st = taosGetTimestampUs(); *handle = pTaskInfo->dsHandle;
pRuntimeEnv->outputBuf = pRuntimeEnv->proot->exec(pRuntimeEnv->proot, &newgroup);
pQInfo->summary.elapsedTime += (taosGetTimestampUs() - st); while(1) {
#ifdef TEST_IMPL st = taosGetTimestampUs();
waitMoment(pQInfo); SSDataBlock* pRes = pTaskInfo->pRoot->exec(pTaskInfo->pRoot, &newgroup);
#endif
publishOperatorProfEvent(pRuntimeEnv->proot, QUERY_PROF_AFTER_OPERATOR_EXEC);
pRuntimeEnv->resultInfo.total += GET_NUM_OF_RESULTS(pRuntimeEnv);
if (isQueryKilled(pQInfo)) {
qDebug("QInfo:0x%"PRIx64" query is killed", pQInfo->qId);
} else if (GET_NUM_OF_RESULTS(pRuntimeEnv) == 0) {
qDebug("QInfo:0x%"PRIx64" over, %u tables queried, total %"PRId64" rows returned", pQInfo->qId, pRuntimeEnv->tableqinfoGroupInfo.numOfTables,
pRuntimeEnv->resultInfo.total);
} else {
qDebug("QInfo:0x%"PRIx64" query paused, %d rows returned, total:%" PRId64 " rows", pQInfo->qId,
GET_NUM_OF_RESULTS(pRuntimeEnv), pRuntimeEnv->resultInfo.total);
}
return doBuildResCheck(pQInfo); pTaskInfo->cost.elapsedTime += (taosGetTimestampUs() - st);
publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pRes == NULL) { // no results generated yet, abort
dsEndPut(pTaskInfo->dsHandle, pTaskInfo->cost.elapsedTime);
return pTaskInfo->code;
}
bool qcontinue = false;
SInputData inputData = {.pData = pRes, .pTableRetrieveTsMap = NULL};
pTaskInfo->code = dsPutDataBlock(pTaskInfo->dsHandle, &inputData, &qcontinue);
if (isTaskKilled(pTaskInfo)) {
qDebug("QInfo:0x%" PRIx64 " task is killed", GET_TASKID(pTaskInfo));
// } else if (GET_NUM_OF_RESULTS(pRuntimeEnv) == 0) {
// qDebug("QInfo:0x%"PRIx64" over, %u tables queried, total %"PRId64" rows returned", pTaskInfo->qId, pRuntimeEnv->tableqinfoGroupInfo.numOfTables,
// pRuntimeEnv->resultInfo.total);
}
if (!qcontinue) {
qDebug("QInfo:0x%"PRIx64" query paused, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d", GET_TASKID(pTaskInfo),
0, 0L, 0);
return pTaskInfo->code;
}
}
} }
int32_t qRetrieveQueryResultInfo(qTaskInfo_t qinfo, bool* buildRes, void* pRspContext) { int32_t qRetrieveQueryResultInfo(qTaskInfo_t qinfo, bool* buildRes, void* pRspContext) {
...@@ -398,13 +284,13 @@ int32_t qKillTask(qTaskInfo_t qinfo) { ...@@ -398,13 +284,13 @@ int32_t qKillTask(qTaskInfo_t qinfo) {
} }
int32_t qIsTaskCompleted(qTaskInfo_t qinfo) { int32_t qIsTaskCompleted(qTaskInfo_t qinfo) {
SQInfo *pQInfo = (SQInfo *)qinfo; SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)qinfo;
if (pQInfo == NULL || !isValidQInfo(pQInfo)) { if (pTaskInfo == NULL /*|| !isValidQInfo(pTaskInfo)*/) {
return TSDB_CODE_QRY_INVALID_QHANDLE; return TSDB_CODE_QRY_INVALID_QHANDLE;
} }
return isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQInfo->runtimeEnv.status, QUERY_OVER); return isTaskKilled(pTaskInfo) || Q_STATUS_EQUAL(pTaskInfo->status, TASK_OVER);
} }
void qDestroyTask(qTaskInfo_t qHandle) { void qDestroyTask(qTaskInfo_t qHandle) {
......
source/libs/executor/src/executorimpl.c
浏览文件 @ 369b5559
...@@ -26,6 +26,7 @@ ...@@ -26,6 +26,7 @@
#include "thash.h" #include "thash.h"
#include "ttypes.h" #include "ttypes.h"
#include "query.h" #include "query.h"
#include "tsdb.h"
#define IS_MAIN_SCAN(runtime) ((runtime)->scanFlag == MAIN_SCAN) #define IS_MAIN_SCAN(runtime) ((runtime)->scanFlag == MAIN_SCAN)
#define IS_REVERSE_SCAN(runtime) ((runtime)->scanFlag == REVERSE_SCAN) #define IS_REVERSE_SCAN(runtime) ((runtime)->scanFlag == REVERSE_SCAN)
...@@ -204,12 +205,10 @@ static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput); ...@@ -204,12 +205,10 @@ static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput);
static void destroyAggOperatorInfo(void* param, int32_t numOfOutput); static void destroyAggOperatorInfo(void* param, int32_t numOfOutput);
static void destroyOperatorInfo(SOperatorInfo* pOperator); static void destroyOperatorInfo(SOperatorInfo* pOperator);
void setTaskStatus(STaskInfo *pTaskInfo, int8_t status);
static void doSetOperatorCompleted(SOperatorInfo* pOperator) { static void doSetOperatorCompleted(SOperatorInfo* pOperator) {
pOperator->status = OP_EXEC_DONE; pOperator->status = OP_EXEC_DONE;
if (pOperator->pTaskInfo != NULL) { if (pOperator->pTaskInfo != NULL) {
setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
} }
} }
...@@ -1509,7 +1508,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn ...@@ -1509,7 +1508,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn
int16_t type = pColInfoData->info.type; int16_t type = pColInfoData->info.type;
if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) {
//qError("QInfo:0x%"PRIx64" group by not supported on double/float columns, abort", GET_QID(pRuntimeEnv)); //qError("QInfo:0x%"PRIx64" group by not supported on double/float columns, abort", GET_TASKID(pRuntimeEnv));
return; return;
} }
...@@ -1908,7 +1907,7 @@ static SQLFunctionCtx* createSQLFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprI ...@@ -1908,7 +1907,7 @@ static SQLFunctionCtx* createSQLFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprI
pCtx->param[2].i = pQueryAttr->window.ekey; pCtx->param[2].i = pQueryAttr->window.ekey;
pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT;
} else if (functionId == FUNCTION_ARITHM) { } else if (functionId == FUNCTION_ARITHM) {
pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i); // pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i);
} }
} }
...@@ -1940,7 +1939,7 @@ static void* destroySQLFunctionCtx(SQLFunctionCtx* pCtx, int32_t numOfOutput) { ...@@ -1940,7 +1939,7 @@ static void* destroySQLFunctionCtx(SQLFunctionCtx* pCtx, int32_t numOfOutput) {
} }
static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfTables, SArray* pOperator, void* merger) { static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfTables, SArray* pOperator, void* merger) {
//qDebug("QInfo:0x%"PRIx64" setup runtime env", GET_QID(pRuntimeEnv)); //qDebug("QInfo:0x%"PRIx64" setup runtime env", GET_TASKID(pRuntimeEnv));
STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr;
pRuntimeEnv->prevGroupId = INT32_MIN; pRuntimeEnv->prevGroupId = INT32_MIN;
...@@ -1977,7 +1976,7 @@ static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfT ...@@ -1977,7 +1976,7 @@ static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfT
} }
} }
//qDebug("QInfo:0x%"PRIx64" init runtime environment completed", GET_QID(pRuntimeEnv)); //qDebug("QInfo:0x%"PRIx64" init runtime environment completed", GET_TASKID(pRuntimeEnv));
// group by normal column, sliding window query, interval query are handled by interval query processor // group by normal column, sliding window query, interval query are handled by interval query processor
// interval (down sampling operation) // interval (down sampling operation)
...@@ -2164,8 +2163,8 @@ _clean: ...@@ -2164,8 +2163,8 @@ _clean:
static void doFreeQueryHandle(STaskRuntimeEnv* pRuntimeEnv) { static void doFreeQueryHandle(STaskRuntimeEnv* pRuntimeEnv) {
STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
// tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); // tsdbCleanupQueryHandle(pRuntimeEnv->pTsdbReadHandle);
pRuntimeEnv->pQueryHandle = NULL; pRuntimeEnv->pTsdbReadHandle = NULL;
// SMemRef* pMemRef = &pQueryAttr->memRef; // SMemRef* pMemRef = &pQueryAttr->memRef;
// assert(pMemRef->ref == 0 && pMemRef->snapshot.imem == NULL && pMemRef->snapshot.mem == NULL); // assert(pMemRef->ref == 0 && pMemRef->snapshot.imem == NULL && pMemRef->snapshot.mem == NULL);
...@@ -2191,7 +2190,7 @@ static void teardownQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv) { ...@@ -2191,7 +2190,7 @@ static void teardownQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv) {
//qDebug("QInfo:0x%"PRIx64" teardown runtime env", pQInfo->qId); //qDebug("QInfo:0x%"PRIx64" teardown runtime env", pQInfo->qId);
destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pQueryAttr->numOfOutput); destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pQueryAttr->numOfOutput);
destroyUdfInfo(pRuntimeEnv->pUdfInfo); // destroyUdfInfo(pRuntimeEnv->pUdfInfo);
destroyResultBuf(pRuntimeEnv->pResultBuf); destroyResultBuf(pRuntimeEnv->pResultBuf);
doFreeQueryHandle(pRuntimeEnv); doFreeQueryHandle(pRuntimeEnv);
...@@ -2224,18 +2223,18 @@ static bool needBuildResAfterQueryComplete(SQInfo* pQInfo) { ...@@ -2224,18 +2223,18 @@ static bool needBuildResAfterQueryComplete(SQInfo* pQInfo) {
return pQInfo->rspContext != NULL; return pQInfo->rspContext != NULL;
} }
bool isQueryKilled(SQInfo *pQInfo) { bool isTaskKilled(SExecTaskInfo *pTaskInfo) {
if (IS_QUERY_KILLED(pQInfo)) { if (IS_QUERY_KILLED(pTaskInfo)) {
return true; return true;
} }
// query has been executed more than tsShellActivityTimer, and the retrieve has not arrived // query has been executed more than tsShellActivityTimer, and the retrieve has not arrived
// abort current query execution. // abort current query execution.
if (pQInfo->owner != 0 && ((taosGetTimestampSec() - pQInfo->startExecTs/1000) > getMaximumIdleDurationSec()) && if (pTaskInfo->owner != 0 && ((taosGetTimestampSec() - pTaskInfo->cost.start/1000) > 10*getMaximumIdleDurationSec())
(!needBuildResAfterQueryComplete(pQInfo))) { /*(!needBuildResAfterQueryComplete(pTaskInfo))*/) {
assert(pQInfo->startExecTs != 0); assert(pTaskInfo->cost.start != 0);
//qDebug("QInfo:%" PRIu64 " retrieve not arrive beyond %d ms, abort current query execution, start:%" PRId64 // qDebug("QInfo:%" PRIu64 " retrieve not arrive beyond %d ms, abort current query execution, start:%" PRId64
// ", current:%d", pQInfo->qId, 1, pQInfo->startExecTs, taosGetTimestampSec()); // ", current:%d", pQInfo->qId, 1, pQInfo->startExecTs, taosGetTimestampSec());
return true; return true;
} }
...@@ -2887,7 +2886,7 @@ void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFi ...@@ -2887,7 +2886,7 @@ void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFi
} }
} }
int32_t loadDataBlock(STaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { int32_t loadDataBlock(SExecTaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) {
STaskCostInfo* pCost = &pTaskInfo->cost; STaskCostInfo* pCost = &pTaskInfo->cost;
pCost->totalBlocks += 1; pCost->totalBlocks += 1;
...@@ -2896,13 +2895,15 @@ int32_t loadDataBlock(STaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDa ...@@ -2896,13 +2895,15 @@ int32_t loadDataBlock(STaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDa
pCost->totalCheckedRows += pBlock->info.rows; pCost->totalCheckedRows += pBlock->info.rows;
pCost->loadBlocks += 1; pCost->loadBlocks += 1;
// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL);
if (pBlock->pDataBlock == NULL) { if (pBlock->pDataBlock == NULL) {
return terrno; return terrno;
} else {
return TSDB_CODE_SUCCESS;
} }
} }
int32_t loadDataBlockOnDemand(STaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { int32_t loadDataBlockOnDemand(SExecTaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) {
*status = BLK_DATA_NO_NEEDED; *status = BLK_DATA_NO_NEEDED;
pBlock->pDataBlock = NULL; pBlock->pDataBlock = NULL;
...@@ -2991,10 +2992,10 @@ int32_t loadDataBlockOnDemand(STaskInfo *pTaskInfo, STableScanInfo* pTableScanIn ...@@ -2991,10 +2992,10 @@ int32_t loadDataBlockOnDemand(STaskInfo *pTaskInfo, STableScanInfo* pTableScanIn
} else if ((*status) == BLK_DATA_STATIS_NEEDED) { } else if ((*status) == BLK_DATA_STATIS_NEEDED) {
// this function never returns error? // this function never returns error?
pCost->loadBlockStatis += 1; pCost->loadBlockStatis += 1;
// tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockAgg); // tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg);
if (pBlock->pBlockAgg == NULL) { // data block statistics does not exist, load data block if (pBlock->pBlockAgg == NULL) { // data block statistics does not exist, load data block
// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); // pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL);
pCost->totalCheckedRows += pBlock->info.rows; pCost->totalCheckedRows += pBlock->info.rows;
} }
} else { } else {
...@@ -3002,7 +3003,7 @@ int32_t loadDataBlockOnDemand(STaskInfo *pTaskInfo, STableScanInfo* pTableScanIn ...@@ -3002,7 +3003,7 @@ int32_t loadDataBlockOnDemand(STaskInfo *pTaskInfo, STableScanInfo* pTableScanIn
// load the data block statistics to perform further filter // load the data block statistics to perform further filter
pCost->loadBlockStatis += 1; pCost->loadBlockStatis += 1;
// tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockAgg); // tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg);
if (pQueryAttr->topBotQuery && pBlock->pBlockAgg != NULL) { if (pQueryAttr->topBotQuery && pBlock->pBlockAgg != NULL) {
{ // set previous window { // set previous window
...@@ -3048,7 +3049,7 @@ int32_t loadDataBlockOnDemand(STaskInfo *pTaskInfo, STableScanInfo* pTableScanIn ...@@ -3048,7 +3049,7 @@ int32_t loadDataBlockOnDemand(STaskInfo *pTaskInfo, STableScanInfo* pTableScanIn
pCost->totalCheckedRows += pBlockInfo->rows; pCost->totalCheckedRows += pBlockInfo->rows;
pCost->loadBlocks += 1; pCost->loadBlocks += 1;
// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); // pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL);
// if (pBlock->pDataBlock == NULL) { // if (pBlock->pDataBlock == NULL) {
// return terrno; // return terrno;
// } // }
...@@ -3449,12 +3450,12 @@ void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size) { ...@@ -3449,12 +3450,12 @@ void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size) {
} }
} }
void setTaskStatus(STaskInfo *pTaskInfo, int8_t status) { void setTaskStatus(SExecTaskInfo *pTaskInfo, int8_t status) {
if (status == QUERY_NOT_COMPLETED) { if (status == TASK_NOT_COMPLETED) {
pTaskInfo->status = status; pTaskInfo->status = status;
} else { } else {
// QUERY_NOT_COMPLETED is not compatible with any other status, so clear its position first // QUERY_NOT_COMPLETED is not compatible with any other status, so clear its position first
CLEAR_QUERY_STATUS(pTaskInfo, QUERY_NOT_COMPLETED); CLEAR_QUERY_STATUS(pTaskInfo, TASK_NOT_COMPLETED);
pTaskInfo->status |= status; pTaskInfo->status |= status;
} }
} }
...@@ -3476,6 +3477,10 @@ static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SQLFunctionCt ...@@ -3476,6 +3477,10 @@ static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SQLFunctionCt
SWITCH_ORDER(pTableScanInfo->order); SWITCH_ORDER(pTableScanInfo->order);
setupQueryRangeForReverseScan(pTableScanInfo); setupQueryRangeForReverseScan(pTableScanInfo);
pTableScanInfo->times = 1;
pTableScanInfo->current = 0;
pTableScanInfo->reverseTimes = 0;
} }
void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) {
...@@ -3700,10 +3705,10 @@ void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprI ...@@ -3700,10 +3705,10 @@ void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprI
// //
// int16_t tagType = pCtx[0].tag.nType; // int16_t tagType = pCtx[0].tag.nType;
// if (tagType == TSDB_DATA_TYPE_BINARY || tagType == TSDB_DATA_TYPE_NCHAR) { // if (tagType == TSDB_DATA_TYPE_BINARY || tagType == TSDB_DATA_TYPE_NCHAR) {
// //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%s", GET_QID(pRuntimeEnv), // //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%s", GET_TASKID(pRuntimeEnv),
//// pExprInfo->base.param[0].i, pCtx[0].tag.pz); //// pExprInfo->base.param[0].i, pCtx[0].tag.pz);
// } else { // } else {
// //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64, GET_QID(pRuntimeEnv), // //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64, GET_TASKID(pRuntimeEnv),
//// pExprInfo->base.param[0].i, pCtx[0].tag.i); //// pExprInfo->base.param[0].i, pCtx[0].tag.i);
// } // }
// } // }
...@@ -3723,9 +3728,9 @@ int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, S ...@@ -3723,9 +3728,9 @@ int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, S
// failed to find data with the specified tag value and vnodeId // failed to find data with the specified tag value and vnodeId
if (!tsBufIsValidElem(&elem)) { if (!tsBufIsValidElem(&elem)) {
if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) {
//qError("QInfo:0x%"PRIx64" failed to find tag:%s in ts_comp", GET_QID(pRuntimeEnv), pTag->pz); //qError("QInfo:0x%"PRIx64" failed to find tag:%s in ts_comp", GET_TASKID(pRuntimeEnv), pTag->pz);
} else { } else {
//qError("QInfo:0x%"PRIx64" failed to find tag:%" PRId64 " in ts_comp", GET_QID(pRuntimeEnv), pTag->i); //qError("QInfo:0x%"PRIx64" failed to find tag:%" PRId64 " in ts_comp", GET_TASKID(pRuntimeEnv), pTag->i);
} }
return -1; return -1;
...@@ -3734,17 +3739,17 @@ int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, S ...@@ -3734,17 +3739,17 @@ int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, S
// Keep the cursor info of current table // Keep the cursor info of current table
pTableQueryInfo->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); pTableQueryInfo->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf);
if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) {
//qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); //qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex);
} else { } else {
//qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex);
} }
} else { } else {
tsBufSetCursor(pRuntimeEnv->pTsBuf, &pTableQueryInfo->cur); tsBufSetCursor(pRuntimeEnv->pTsBuf, &pTableQueryInfo->cur);
if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) {
//qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); //qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex);
} else { } else {
//qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex);
} }
} }
...@@ -3882,7 +3887,7 @@ static int32_t doCopyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, SGroupResInfo* p ...@@ -3882,7 +3887,7 @@ static int32_t doCopyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, SGroupResInfo* p
int32_t start = 0; int32_t start = 0;
int32_t step = -1; int32_t step = -1;
//qDebug("QInfo:0x%"PRIx64" start to copy data from windowResInfo to output buf", GET_QID(pRuntimeEnv)); //qDebug("QInfo:0x%"PRIx64" start to copy data from windowResInfo to output buf", GET_TASKID(pRuntimeEnv));
assert(orderType == TSDB_ORDER_ASC || orderType == TSDB_ORDER_DESC); assert(orderType == TSDB_ORDER_ASC || orderType == TSDB_ORDER_DESC);
if (orderType == TSDB_ORDER_ASC) { if (orderType == TSDB_ORDER_ASC) {
...@@ -3927,7 +3932,7 @@ static int32_t doCopyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, SGroupResInfo* p ...@@ -3927,7 +3932,7 @@ static int32_t doCopyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, SGroupResInfo* p
} }
} }
//qDebug("QInfo:0x%"PRIx64" copy data to query buf completed", GET_QID(pRuntimeEnv)); //qDebug("QInfo:0x%"PRIx64" copy data to query buf completed", GET_TASKID(pRuntimeEnv));
pBlock->info.rows = numOfResult; pBlock->info.rows = numOfResult;
return 0; return 0;
} }
...@@ -4055,7 +4060,7 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data ...@@ -4055,7 +4060,7 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data
//qDebug("QInfo:0x%"PRIx64" set %d subscribe info", pQInfo->qId, total); //qDebug("QInfo:0x%"PRIx64" set %d subscribe info", pQInfo->qId, total);
// Check if query is completed or not for stable query or normal table query respectively. // Check if query is completed or not for stable query or normal table query respectively.
if (Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED) && pRuntimeEnv->proot->status == OP_EXEC_DONE) { if (Q_STATUS_EQUAL(pRuntimeEnv->status, TASK_COMPLETED) && pRuntimeEnv->proot->status == OP_EXEC_DONE) {
// setTaskStatus(pOperator->pTaskInfo, QUERY_OVER); // setTaskStatus(pOperator->pTaskInfo, QUERY_OVER);
} }
} }
...@@ -4232,7 +4237,7 @@ void queryCostStatis(SQInfo *pQInfo) { ...@@ -4232,7 +4237,7 @@ void queryCostStatis(SQInfo *pQInfo) {
// //
// assert(pQueryAttr->pos >= 0 && pQueryAttr->pos <= pBlockInfo->rows - 1); // assert(pQueryAttr->pos >= 0 && pQueryAttr->pos <= pBlockInfo->rows - 1);
// //
// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); // SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL);
// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0);
// //
// // update the pQueryAttr->limit.offset value, and pQueryAttr->pos value // // update the pQueryAttr->limit.offset value, and pQueryAttr->pos value
...@@ -4244,7 +4249,7 @@ void queryCostStatis(SQInfo *pQInfo) { ...@@ -4244,7 +4249,7 @@ void queryCostStatis(SQInfo *pQInfo) {
// //
// int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); // int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock);
// //
// //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numBlocksOfStep:%d, numOfRes:%d, lastKey:%"PRId64, GET_QID(pRuntimeEnv), // //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numBlocksOfStep:%d, numOfRes:%d, lastKey:%"PRId64, GET_TASKID(pRuntimeEnv),
// pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, pQuery->current->lastKey); // pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, pQuery->current->lastKey);
//} //}
...@@ -4259,22 +4264,22 @@ void queryCostStatis(SQInfo *pQInfo) { ...@@ -4259,22 +4264,22 @@ void queryCostStatis(SQInfo *pQInfo) {
// int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); // int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order);
// //
// STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; // STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current;
// TsdbQueryHandleT pQueryHandle = pRuntimeEnv->pQueryHandle; // TsdbQueryHandleT pTsdbReadHandle = pRuntimeEnv->pTsdbReadHandle;
// //
// SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; // SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER;
// while (tsdbNextDataBlock(pQueryHandle)) { // while (tsdbNextDataBlock(pTsdbReadHandle)) {
// if (isQueryKilled(pRuntimeEnv->qinfo)) { // if (isTaskKilled(pRuntimeEnv->qinfo)) {
// longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); // longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED);
// } // }
// //
// tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); // tsdbRetrieveDataBlockInfo(pTsdbReadHandle, &blockInfo);
// //
// if (pQueryAttr->limit.offset > blockInfo.rows) { // if (pQueryAttr->limit.offset > blockInfo.rows) {
// pQueryAttr->limit.offset -= blockInfo.rows; // pQueryAttr->limit.offset -= blockInfo.rows;
// pTableQueryInfo->lastKey = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? blockInfo.window.ekey : blockInfo.window.skey; // pTableQueryInfo->lastKey = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? blockInfo.window.ekey : blockInfo.window.skey;
// pTableQueryInfo->lastKey += step; // pTableQueryInfo->lastKey += step;
// //
// //qDebug("QInfo:0x%"PRIx64" skip rows:%d, offset:%" PRId64, GET_QID(pRuntimeEnv), blockInfo.rows, // //qDebug("QInfo:0x%"PRIx64" skip rows:%d, offset:%" PRId64, GET_TASKID(pRuntimeEnv), blockInfo.rows,
// pQuery->limit.offset); // pQuery->limit.offset);
// } else { // find the appropriated start position in current block // } else { // find the appropriated start position in current block
// updateOffsetVal(pRuntimeEnv, &blockInfo); // updateOffsetVal(pRuntimeEnv, &blockInfo);
...@@ -4300,7 +4305,7 @@ void queryCostStatis(SQInfo *pQInfo) { ...@@ -4300,7 +4305,7 @@ void queryCostStatis(SQInfo *pQInfo) {
// //
// // load the data block and check data remaining in current data block // // load the data block and check data remaining in current data block
// // TODO optimize performance // // TODO optimize performance
// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); // SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL);
// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0);
// //
// tw = *win; // tw = *win;
...@@ -4323,7 +4328,7 @@ void queryCostStatis(SQInfo *pQInfo) { ...@@ -4323,7 +4328,7 @@ void queryCostStatis(SQInfo *pQInfo) {
// pRuntimeEnv->resultRowInfo.curIndex = index; // restore the window index // pRuntimeEnv->resultRowInfo.curIndex = index; // restore the window index
// //
// //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%" PRId64, // //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%" PRId64,
// GET_QID(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, // GET_TASKID(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes,
// pQueryAttr->current->lastKey); // pQueryAttr->current->lastKey);
// //
// return key; // return key;
...@@ -4365,8 +4370,8 @@ void queryCostStatis(SQInfo *pQInfo) { ...@@ -4365,8 +4370,8 @@ void queryCostStatis(SQInfo *pQInfo) {
// STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; // STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current;
// //
// SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; // SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER;
// while (tsdbNextDataBlock(pRuntimeEnv->pQueryHandle)) { // while (tsdbNextDataBlock(pRuntimeEnv->pTsdbReadHandle)) {
// tsdbRetrieveDataBlockInfo(pRuntimeEnv->pQueryHandle, &blockInfo); // tsdbRetrieveDataBlockInfo(pRuntimeEnv->pTsdbReadHandle, &blockInfo);
// //
// if (QUERY_IS_ASC_QUERY(pQueryAttr)) { // if (QUERY_IS_ASC_QUERY(pQueryAttr)) {
// if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { // if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) {
...@@ -4412,7 +4417,7 @@ void queryCostStatis(SQInfo *pQInfo) { ...@@ -4412,7 +4417,7 @@ void queryCostStatis(SQInfo *pQInfo) {
// */ // */
// if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) { // if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) {
// //
// SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); // SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL);
// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0);
// //
// if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) { // if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) {
...@@ -4486,7 +4491,7 @@ static int32_t setupQueryHandle(void* tsdb, STaskRuntimeEnv* pRuntimeEnv, int64_ ...@@ -4486,7 +4491,7 @@ static int32_t setupQueryHandle(void* tsdb, STaskRuntimeEnv* pRuntimeEnv, int64_
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
if (isFirstLastRowQuery(pQueryAttr)) { if (isFirstLastRowQuery(pQueryAttr)) {
pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); pRuntimeEnv->pTsdbReadHandle = tsdbQueryLastRow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef);
// update the query time window // update the query time window
pQueryAttr->window = cond.twindow; pQueryAttr->window = cond.twindow;
...@@ -4507,11 +4512,11 @@ static int32_t setupQueryHandle(void* tsdb, STaskRuntimeEnv* pRuntimeEnv, int64_ ...@@ -4507,11 +4512,11 @@ static int32_t setupQueryHandle(void* tsdb, STaskRuntimeEnv* pRuntimeEnv, int64_
} }
} }
} else if (isCachedLastQuery(pQueryAttr)) { } else if (isCachedLastQuery(pQueryAttr)) {
pRuntimeEnv->pQueryHandle = tsdbQueryCacheLast(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); pRuntimeEnv->pTsdbReadHandle = tsdbQueryCacheLast(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef);
} else if (pQueryAttr->pointInterpQuery) { } else if (pQueryAttr->pointInterpQuery) {
pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); pRuntimeEnv->pTsdbReadHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef);
} else { } else {
pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); pRuntimeEnv->pTsdbReadHandle = tsdbQueryTables(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef);
} }
#endif #endif
return terrno; return terrno;
...@@ -4543,19 +4548,19 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr ...@@ -4543,19 +4548,19 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr
switch(tbScanner) { switch(tbScanner) {
// case OP_TableBlockInfoScan: { // case OP_TableBlockInfoScan: {
// pRuntimeEnv->proot = createTableBlockInfoScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); // pRuntimeEnv->proot = createTableBlockInfoScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv);
// break; // break;
// } // }
// case OP_TableSeqScan: { // case OP_TableSeqScan: {
// pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); // pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv);
// break; // break;
// } // }
// case OP_DataBlocksOptScan: { // case OP_DataBlocksOptScan: {
// pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0); // pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0);
// break; // break;
// } // }
// case OP_TableScan: { // case OP_TableScan: {
// pRuntimeEnv->proot = createTableScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr)); // pRuntimeEnv->proot = createTableScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr));
// break; // break;
// } // }
default: { // do nothing default: { // do nothing
...@@ -4606,7 +4611,7 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr ...@@ -4606,7 +4611,7 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static void doTableQueryInfoTimeWindowCheck(STaskInfo* pTaskInfo, STableQueryInfo* pTableQueryInfo, int32_t order) { static void doTableQueryInfoTimeWindowCheck(SExecTaskInfo* pTaskInfo, STableQueryInfo* pTableQueryInfo, int32_t order) {
if (order == TSDB_ORDER_ASC) { if (order == TSDB_ORDER_ASC) {
assert( assert(
(pTableQueryInfo->win.skey <= pTableQueryInfo->win.ekey) && (pTableQueryInfo->win.skey <= pTableQueryInfo->win.ekey) &&
...@@ -4679,20 +4684,20 @@ static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { ...@@ -4679,20 +4684,20 @@ static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) {
SOperatorInfo *pOperator = (SOperatorInfo*) param; SOperatorInfo *pOperator = (SOperatorInfo*) param;
STableScanInfo *pTableScanInfo = pOperator->info; STableScanInfo *pTableScanInfo = pOperator->info;
STaskInfo *pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo;
SSDataBlock *pBlock = &pTableScanInfo->block; SSDataBlock *pBlock = &pTableScanInfo->block;
STableGroupInfo *pTableGroupInfo = &pOperator->pTaskInfo->tableqinfoGroupInfo; STableGroupInfo *pTableGroupInfo = &pOperator->pTaskInfo->tableqinfoGroupInfo;
*newgroup = false; *newgroup = false;
while (/*tsdbNextDataBlock(pTableScanInfo->pQueryHandle)*/1) { while (tsdbNextDataBlock(pTableScanInfo->pTsdbReadHandle)) {
if (isQueryKilled(pOperator->pRuntimeEnv->qinfo)) { if (isTaskKilled(pOperator->pTaskInfo)) {
longjmp(pOperator->pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); longjmp(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
} }
pTableScanInfo->numOfBlocks += 1; pTableScanInfo->numOfBlocks += 1;
// tsdbRetrieveDataBlockInfo(pTableScanInfo->pQueryHandle, &pBlock->info); tsdbRetrieveDataBlockInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->info);
// todo opt // todo opt
// if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) { // if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) {
...@@ -4711,7 +4716,7 @@ static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { ...@@ -4711,7 +4716,7 @@ static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) {
int32_t code = loadDataBlock(pTaskInfo, pTableScanInfo, pBlock, &status); int32_t code = loadDataBlock(pTaskInfo, pTableScanInfo, pBlock, &status);
// int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status); // int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
longjmp(pOperator->pRuntimeEnv->env, code); longjmp(pOperator->pTaskInfo->env, code);
} }
// current block is ignored according to filter result by block statistics data, continue load the next block // current block is ignored according to filter result by block statistics data, continue load the next block
...@@ -4729,7 +4734,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { ...@@ -4729,7 +4734,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) {
SOperatorInfo* pOperator = (SOperatorInfo*) param; SOperatorInfo* pOperator = (SOperatorInfo*) param;
STableScanInfo *pTableScanInfo = pOperator->info; STableScanInfo *pTableScanInfo = pOperator->info;
STaskInfo *pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo;
SResultRowInfo* pResultRowInfo = pTableScanInfo->pResultRowInfo; SResultRowInfo* pResultRowInfo = pTableScanInfo->pResultRowInfo;
*newgroup = false; *newgroup = false;
...@@ -4741,7 +4746,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { ...@@ -4741,7 +4746,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) {
} }
if (++pTableScanInfo->current >= pTableScanInfo->times) { if (++pTableScanInfo->current >= pTableScanInfo->times) {
if (pTableScanInfo->reverseTimes <= 0/* || isTsdbCacheLastRow(pTableScanInfo->pQueryHandle)*/) { if (pTableScanInfo->reverseTimes <= 0/* || isTsdbCacheLastRow(pTableScanInfo->pTsdbReadHandle)*/) {
return NULL; return NULL;
} else { } else {
break; break;
...@@ -4750,9 +4755,9 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { ...@@ -4750,9 +4755,9 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) {
// do prepare for the next round table scan operation // do prepare for the next round table scan operation
// STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); // STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window);
// tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); // tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond);
setTaskStatus(pTaskInfo, QUERY_NOT_COMPLETED); setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
pTableScanInfo->scanFlag = REPEAT_SCAN; pTableScanInfo->scanFlag = REPEAT_SCAN;
// if (pTaskInfo->pTsBuf) { // if (pTaskInfo->pTsBuf) {
...@@ -4764,8 +4769,8 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { ...@@ -4764,8 +4769,8 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) {
pResultRowInfo->curPos = 0; pResultRowInfo->curPos = 0;
} }
//qDebug("QInfo:0x%"PRIx64" start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, qDebug("QInfo:0x%"PRIx64" start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64,
// GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey); GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey);
} }
SSDataBlock *p = NULL; SSDataBlock *p = NULL;
...@@ -4773,15 +4778,10 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { ...@@ -4773,15 +4778,10 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) {
if (pTableScanInfo->reverseTimes > 0) { if (pTableScanInfo->reverseTimes > 0) {
setupEnvForReverseScan(pTableScanInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); setupEnvForReverseScan(pTableScanInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput);
// STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); // STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window);
// tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); // tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond);
//qDebug("QInfo:0x%"PRIx64" start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64,
// GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey);
pTableScanInfo->times = 1; qDebug("QInfo:0x%"PRIx64" start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64,
pTableScanInfo->current = 0; GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey);
pTableScanInfo->reverseTimes = 0;
// pTableScanInfo->order = cond.order;
if (pResultRowInfo->size > 0) { if (pResultRowInfo->size > 0) {
pResultRowInfo->curPos = pResultRowInfo->size - 1; pResultRowInfo->curPos = pResultRowInfo->size - 1;
...@@ -4814,8 +4814,8 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { ...@@ -4814,8 +4814,8 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) {
tableBlockDist.maxRows = INT_MIN; tableBlockDist.maxRows = INT_MIN;
tableBlockDist.minRows = INT_MAX; tableBlockDist.minRows = INT_MAX;
tsdbGetFileBlocksDistInfo(pTableScanInfo->pQueryHandle, &tableBlockDist); tsdbGetFileBlocksDistInfo(pTableScanInfo->pTsdbReadHandle, &tableBlockDist);
tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pQueryHandle); tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pTsdbReadHandle);
SSDataBlock* pBlock = &pTableScanInfo->block; SSDataBlock* pBlock = &pTableScanInfo->block;
pBlock->info.rows = 1; pBlock->info.rows = 1;
...@@ -4842,11 +4842,11 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { ...@@ -4842,11 +4842,11 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) {
} }
SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime) { SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo) {
assert(repeatTime > 0 && numOfOutput > 0); assert(repeatTime > 0 && numOfOutput > 0);
STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo));
pInfo->pQueryHandle = pTsdbQueryHandle; pInfo->pTsdbReadHandle = pTsdbQueryHandle;
pInfo->times = repeatTime; pInfo->times = repeatTime;
pInfo->reverseTimes = 0; pInfo->reverseTimes = 0;
pInfo->order = order; pInfo->order = order;
...@@ -4862,6 +4862,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, in ...@@ -4862,6 +4862,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, in
pOperator->numOfOutput = numOfOutput; pOperator->numOfOutput = numOfOutput;
pOperator->pRuntimeEnv = NULL; pOperator->pRuntimeEnv = NULL;
pOperator->exec = doTableScan; pOperator->exec = doTableScan;
pOperator->pTaskInfo = pTaskInfo;
return pOperator; return pOperator;
} }
...@@ -4869,7 +4870,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, in ...@@ -4869,7 +4870,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, in
SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv) { SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv) {
STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo));
pInfo->pQueryHandle = pTsdbQueryHandle; pInfo->pTsdbReadHandle = pTsdbQueryHandle;
pInfo->times = 1; pInfo->times = 1;
pInfo->reverseTimes = 0; pInfo->reverseTimes = 0;
pInfo->order = pRuntimeEnv->pQueryAttr->order.order; pInfo->order = pRuntimeEnv->pQueryAttr->order.order;
...@@ -4893,7 +4894,7 @@ SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEn ...@@ -4893,7 +4894,7 @@ SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEn
SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv) { SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv) {
STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo));
pInfo->pQueryHandle = pTsdbQueryHandle; pInfo->pTsdbReadHandle = pTsdbQueryHandle;
pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
SColumnInfoData infoData = {{0}}; SColumnInfoData infoData = {{0}};
...@@ -4977,7 +4978,7 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, STaskRuntimeE ...@@ -4977,7 +4978,7 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, STaskRuntimeE
assert(repeatTime > 0); assert(repeatTime > 0);
STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo));
pInfo->pQueryHandle = pTsdbQueryHandle; pInfo->pTsdbReadHandle = pTsdbQueryHandle;
pInfo->times = repeatTime; pInfo->times = repeatTime;
pInfo->reverseTimes = reverseTime; pInfo->reverseTimes = reverseTime;
pInfo->current = 0; pInfo->current = 0;
...@@ -5145,7 +5146,7 @@ SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S ...@@ -5145,7 +5146,7 @@ SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S
pOperator->numOfOutput = numOfOutput; pOperator->numOfOutput = numOfOutput;
pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->exec = doGlobalAggregate; // pOperator->exec = doGlobalAggregate;
pOperator->cleanup = destroyGlobalAggOperatorInfo; pOperator->cleanup = destroyGlobalAggOperatorInfo;
appendUpstream(pOperator, downstream); appendUpstream(pOperator, downstream);
...@@ -5188,7 +5189,7 @@ SOperatorInfo *createMultiwaySortOperatorInfo(STaskRuntimeEnv *pRuntimeEnv, SExp ...@@ -5188,7 +5189,7 @@ SOperatorInfo *createMultiwaySortOperatorInfo(STaskRuntimeEnv *pRuntimeEnv, SExp
pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->numOfOutput = numOfOutput; pOperator->numOfOutput = numOfOutput;
pOperator->pExpr = pExpr; pOperator->pExpr = pExpr;
pOperator->exec = doMultiwayMergeSort; // pOperator->exec = doMultiwayMergeSort;
pOperator->cleanup = destroyGlobalAggOperatorInfo; pOperator->cleanup = destroyGlobalAggOperatorInfo;
return pOperator; return pOperator;
} }
...@@ -5476,7 +5477,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { ...@@ -5476,7 +5477,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) {
assert(*newgroup == false); assert(*newgroup == false);
*newgroup = prevVal; *newgroup = prevVal;
setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
break; break;
} }
...@@ -5644,7 +5645,7 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { ...@@ -5644,7 +5645,7 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) {
pOperator->status = OP_RES_TO_RETURN; pOperator->status = OP_RES_TO_RETURN;
closeAllResultRows(&pIntervalInfo->resultRowInfo); closeAllResultRows(&pIntervalInfo->resultRowInfo);
setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset);
initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo);
...@@ -5704,7 +5705,7 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { ...@@ -5704,7 +5705,7 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) {
pOperator->status = OP_RES_TO_RETURN; pOperator->status = OP_RES_TO_RETURN;
closeAllResultRows(&pIntervalInfo->resultRowInfo); closeAllResultRows(&pIntervalInfo->resultRowInfo);
setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset);
initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo);
...@@ -5767,7 +5768,7 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { ...@@ -5767,7 +5768,7 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) {
pOperator->status = OP_RES_TO_RETURN; pOperator->status = OP_RES_TO_RETURN;
pQueryAttr->order.order = order; // TODO : restore the order pQueryAttr->order.order = order; // TODO : restore the order
doCloseAllTimeWindow(pRuntimeEnv); doCloseAllTimeWindow(pRuntimeEnv);
setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset);
if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) {
...@@ -5822,7 +5823,7 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { ...@@ -5822,7 +5823,7 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) {
pOperator->status = OP_RES_TO_RETURN; pOperator->status = OP_RES_TO_RETURN;
pQueryAttr->order.order = order; // TODO : restore the order pQueryAttr->order.order = order; // TODO : restore the order
doCloseAllTimeWindow(pRuntimeEnv); doCloseAllTimeWindow(pRuntimeEnv);
setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset);
...@@ -5956,7 +5957,7 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { ...@@ -5956,7 +5957,7 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) {
pOperator->status = OP_RES_TO_RETURN; pOperator->status = OP_RES_TO_RETURN;
closeAllResultRows(&pBInfo->resultRowInfo); closeAllResultRows(&pBInfo->resultRowInfo);
setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
finalizeQueryResult(pOperator, pBInfo->pCtx, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); finalizeQueryResult(pOperator, pBInfo->pCtx, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset);
initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo);
...@@ -6094,7 +6095,7 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { ...@@ -6094,7 +6095,7 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) {
static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo *pInfo, STaskRuntimeEnv* pRuntimeEnv, bool* newgroup) { static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo *pInfo, STaskRuntimeEnv* pRuntimeEnv, bool* newgroup) {
pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows; pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows;
int64_t ekey = Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED)?pRuntimeEnv->pQueryAttr->window.ekey:pInfo->existNewGroupBlock->info.window.ekey; int64_t ekey = Q_STATUS_EQUAL(pRuntimeEnv->status, TASK_COMPLETED)?pRuntimeEnv->pQueryAttr->window.ekey:pInfo->existNewGroupBlock->info.window.ekey;
taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo)); taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo));
taosFillSetStartInfo(pInfo->pFillInfo, pInfo->existNewGroupBlock->info.rows, ekey); taosFillSetStartInfo(pInfo->pFillInfo, pInfo->existNewGroupBlock->info.rows, ekey);
...@@ -6422,7 +6423,7 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI ...@@ -6422,7 +6423,7 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI
SFilterOperatorInfo* pInfo = calloc(1, sizeof(SFilterOperatorInfo)); SFilterOperatorInfo* pInfo = calloc(1, sizeof(SFilterOperatorInfo));
assert(numOfFilter > 0 && pCols != NULL); assert(numOfFilter > 0 && pCols != NULL);
doCreateFilterInfo(pCols, numOfOutput, numOfFilter, &pInfo->pFilterInfo, 0); // doCreateFilterInfo(pCols, numOfOutput, numOfFilter, &pInfo->pFilterInfo, 0);
pInfo->numOfFilterCols = numOfFilter; pInfo->numOfFilterCols = numOfFilter;
SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
...@@ -6714,10 +6715,10 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI ...@@ -6714,10 +6715,10 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI
SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
pOperator->name = "SLimitOperator"; pOperator->name = "SLimitOperator";
// pOperator->operatorType = OP_SLimit; pOperator->operatorType = OP_SLimit;
pOperator->blockingOptr = false; pOperator->blockingOptr = false;
pOperator->status = OP_IN_EXECUTING; pOperator->status = OP_IN_EXECUTING;
pOperator->exec = doSLimit; // pOperator->exec = doSLimit;
pOperator->info = pInfo; pOperator->info = pInfo;
pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->cleanup = destroySlimitOperatorInfo; pOperator->cleanup = destroySlimitOperatorInfo;
...@@ -6799,14 +6800,14 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { ...@@ -6799,14 +6800,14 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) {
count += 1; count += 1;
} }
//qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_QID(pRuntimeEnv), count); //qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_TASKID(pRuntimeEnv), count);
} else if (functionId == FUNCTION_COUNT) {// handle the "count(tbname)" query } else if (functionId == FUNCTION_COUNT) {// handle the "count(tbname)" query
SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0);
*(int64_t*)pColInfo->pData = pInfo->totalTables; *(int64_t*)pColInfo->pData = pInfo->totalTables;
count = 1; count = 1;
pOperator->status = OP_EXEC_DONE; pOperator->status = OP_EXEC_DONE;
//qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_QID(pRuntimeEnv), count); //qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_TASKID(pRuntimeEnv), count);
} else { // return only the tags|table name etc. } else { // return only the tags|table name etc.
SExprInfo* pExprInfo = &pOperator->pExpr[0]; // todo use the column list instead of exprinfo SExprInfo* pExprInfo = &pOperator->pExpr[0]; // todo use the column list instead of exprinfo
...@@ -6845,11 +6846,11 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { ...@@ -6845,11 +6846,11 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) {
pOperator->status = OP_EXEC_DONE; pOperator->status = OP_EXEC_DONE;
} }
//qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_QID(pRuntimeEnv), count); //qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count);
} }
if (pOperator->status == OP_EXEC_DONE) { if (pOperator->status == OP_EXEC_DONE) {
setTaskStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); setTaskStatus(pOperator->pRuntimeEnv, TASK_COMPLETED);
} }
pRes->info.rows = count; pRes->info.rows = count;
...@@ -7174,90 +7175,53 @@ static int32_t deserializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t ...@@ -7174,90 +7175,53 @@ static int32_t deserializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
/** static SExecTaskInfo* createExecTaskInfo(uint64_t queryId) {
* { SExecTaskInfo* pTaskInfo = calloc(1, sizeof(SExecTaskInfo));
"Id": { setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
"QueryId": 20,
"TemplateId": 0,
"SubplanId": 0
},
"Node": {
"Name": "TableScan",
"InputSchema": [{
"Type": 9,
"ColId": 1,
"Bytes": 8
}, {
"Type": 4,
"ColId": 2,
"Bytes": 4
}, {
"Type": 8,
"ColId": 3,
"Bytes": 20
}],
"TableScan": {
"TableId": 1,
"TableType": 3,
"Flag": 0,
"Window": {
"StartKey": 0,
"EndKey": 0
}
}
},
"DataSink": {
"Name": "Dispatch",
"Dispatch": {
}
}
}
*/
int32_t parseTaskInfo(const char* msg, int32_t len) {
cJSON* pJson = cJSON_Parse(msg);
if (NULL == pJson) {
return TSDB_CODE_INVALID_MSG;
}
cJSON* pSub = cJSON_GetObjectItem(pJson, "ID");
if (NULL != pSub) {
printf("Id : %s\n", pSub->valuestring);
}
cJSON* pNode = cJSON_GetObjectItem(pJson, "Node"); pthread_mutex_init(&pTaskInfo->lock, NULL);
if (pNode == NULL) { pTaskInfo->cost.created = taosGetTimestampMs();
return TSDB_CODE_INVALID_MSG; pTaskInfo->id.queryId = queryId;
} return pTaskInfo;
}
cJSON* pNodeName = cJSON_GetObjectItem(pNode, "name"); SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTaskInfo, void* param) {
if (pNodeName == NULL) { if (pPhyNode->pChildren == NULL || taosArrayGetSize(pPhyNode->pChildren) == 0) {
return TSDB_CODE_INVALID_MSG; if (pPhyNode->info.type == OP_TableScan) {
size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets);
SOperatorInfo* pOperatorInfo = createTableScanOperator(param, TSDB_ORDER_ASC, numOfCols, 1, pTaskInfo);
pTaskInfo->pRoot = pOperatorInfo;
}
} }
}
printf("node name is: %s\n", pNodeName->valuestring); int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle) {
STsdbQueryCond cond = {.order = TSDB_ORDER_ASC, .numOfCols = 2, .loadExternalRows = false};
cond.twindow.skey = INT64_MIN;
cond.twindow.ekey = INT64_MAX;
cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo));
cJSON* pNodeSchema = cJSON_GetObjectItem(pNode, "InputSchema"); // todo set the correct table column info
if (pNodeSchema == NULL) { cond.colList[0].type = TSDB_DATA_TYPE_TIMESTAMP;
return TSDB_CODE_INVALID_MSG; cond.colList[0].bytes = sizeof(uint64_t);
} cond.colList[0].colId = 1;
cJSON* pOperator = cJSON_GetObjectItem(pNode, pNodeName->valuestring); cond.colList[1].type = TSDB_DATA_TYPE_INT;
if (pOperator == NULL) { cond.colList[1].bytes = sizeof(int32_t);
return TSDB_CODE_INVALID_MSG; cond.colList[1].colId = 2;
}
cJSON* pTableId = cJSON_GetObjectItem(pOperator, "tableId"); STableGroupInfo group = {.numOfTables = 1, .pGroupList = taosArrayInit(1, POINTER_BYTES)};
if (pTableId == NULL) { SArray* pa = taosArrayInit(1, sizeof(STableKeyInfo));
return TSDB_CODE_INVALID_MSG; STableKeyInfo info = {.pTable = NULL, .lastKey = 0, .uid = 1};
} taosArrayPush(pa, &info);
cJSON* pTimeWindow = cJSON_GetObjectItem(pOperator, "window");
if (pTimeWindow == NULL) {
return TSDB_CODE_INVALID_MSG;
}
taosArrayPush(group.pGroupList, &pa);
*pTaskInfo = createExecTaskInfo((uint64_t)pPlan->id.queryId);
tsdbReadHandleT tsdbReadHandle = tsdbQueryTables(readerHandle, &cond, &group, (*pTaskInfo)->id.queryId, NULL);
doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, tsdbReadHandle);
return TSDB_CODE_SUCCESS;
} }
/** /**
...@@ -7704,7 +7668,7 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp ...@@ -7704,7 +7668,7 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp
*pExprInfo = NULL; *pExprInfo = NULL;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
code = initUdfInfo(pUdfInfo); // code = initUdfInfo(pUdfInfo);
if (code) { if (code) {
return code; return code;
} }
...@@ -7996,8 +7960,8 @@ int32_t createFilterInfo(STaskAttr* pQueryAttr, uint64_t qId) { ...@@ -7996,8 +7960,8 @@ int32_t createFilterInfo(STaskAttr* pQueryAttr, uint64_t qId) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
doCreateFilterInfo(pQueryAttr->tableCols, pQueryAttr->numOfCols, pQueryAttr->numOfFilterCols, // doCreateFilterInfo(pQueryAttr->tableCols, pQueryAttr->numOfCols, pQueryAttr->numOfFilterCols,
&pQueryAttr->pFilterInfo, qId); // &pQueryAttr->pFilterInfo, qId);
pQueryAttr->createFilterOperator = true; pQueryAttr->createFilterOperator = true;
...@@ -8505,7 +8469,7 @@ int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t ...@@ -8505,7 +8469,7 @@ int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t
} }
// all data returned, set query over // all data returned, set query over
if (Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED)) { if (Q_STATUS_EQUAL(pRuntimeEnv->status, TASK_COMPLETED)) {
// setTaskStatus(pOperator->pTaskInfo, QUERY_OVER); // setTaskStatus(pOperator->pTaskInfo, QUERY_OVER);
} }
} else { } else {
......
source/libs/executor/test/CMakeLists.txt 0 → 100644
浏览文件 @ 369b5559
MESSAGE(STATUS "build parser unit test")
# GoogleTest requires at least C++11
SET(CMAKE_CXX_STANDARD 11)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(executorTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(
executorTest
PUBLIC os util common transport gtest taos qcom executor function planner
)
TARGET_INCLUDE_DIRECTORIES(
executorTest
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/executor/"
PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/executor/inc"
)
source/libs/executor/test/executorTests.cpp
浏览文件 @ 369b5559
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <executorimpl.h>
#include <gtest/gtest.h>
#include <tglobal.h>
#include <iostream>
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
#include "os.h"
#include "taos.h"
#include "tdef.h"
#include "tvariant.h"
#include "tep.h"
#include "trpc.h"
#include "stub.h"
#include "executor.h"
/**
{
"Id": {
"QueryId": 1.3108161807422521e+19,
"TemplateId": 0,
"SubplanId": 0
},
"Node": {
"Name": "TableScan",
"Targets": [{
"Base": {
"Schema": {
"Type": 9,
"ColId": 5000,
"Bytes": 8
},
"Columns": [{
"TableId": 1,
"Flag": 0,
"Info": {
"ColId": 1,
"Type": 9,
"Bytes": 8
}
}],
"InterBytes": 0
},
"Expr": {
"Type": 4,
"Column": {
"Type": 9,
"ColId": 1,
"Bytes": 8
}
}
}, {
"Base": {
"Schema": {
"Type": 4,
"ColId": 5001,
"Bytes": 4
},
"Columns": [{
"TableId": 1,
"Flag": 0,
"Info": {
"ColId": 2,
"Type": 4,
"Bytes": 4
}
}],
"InterBytes": 0
},
"Expr": {
"Type": 4,
"Column": {
"Type": 4,
"ColId": 2,
"Bytes": 4
}
}
}],
"InputSchema": [{
"Type": 9,
"ColId": 5000,
"Bytes": 8
}, {
"Type": 4,
"ColId": 5001,
"Bytes": 4
}],
"TableScan": {
"TableId": 1,
"TableType": 2,
"Flag": 0,
"Window": {
"StartKey": -9.2233720368547758e+18,
"EndKey": 9.2233720368547758e+18
}
}
},
"DataSink": {
"Name": "Dispatch",
"Dispatch": {
}
}
}
*/
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
TEST(testCase, build_executor_tree_Test) {
const char* msg = "{\n"
"\t\"Id\":\t{\n"
"\t\t\"QueryId\":\t1.3108161807422521e+19,\n"
"\t\t\"TemplateId\":\t0,\n"
"\t\t\"SubplanId\":\t0\n"
"\t},\n"
"\t\"Node\":\t{\n"
"\t\t\"Name\":\t\"TableScan\",\n"
"\t\t\"Targets\":\t[{\n"
"\t\t\t\t\"Base\":\t{\n"
"\t\t\t\t\t\"Schema\":\t{\n"
"\t\t\t\t\t\t\"Type\":\t9,\n"
"\t\t\t\t\t\t\"ColId\":\t5000,\n"
"\t\t\t\t\t\t\"Bytes\":\t8\n"
"\t\t\t\t\t},\n"
"\t\t\t\t\t\"Columns\":\t[{\n"
"\t\t\t\t\t\t\t\"TableId\":\t1,\n"
"\t\t\t\t\t\t\t\"Flag\":\t0,\n"
"\t\t\t\t\t\t\t\"Info\":\t{\n"
"\t\t\t\t\t\t\t\t\"ColId\":\t1,\n"
"\t\t\t\t\t\t\t\t\"Type\":\t9,\n"
"\t\t\t\t\t\t\t\t\"Bytes\":\t8\n"
"\t\t\t\t\t\t\t}\n"
"\t\t\t\t\t\t}],\n"
"\t\t\t\t\t\"InterBytes\":\t0\n"
"\t\t\t\t},\n"
"\t\t\t\t\"Expr\":\t{\n"
"\t\t\t\t\t\"Type\":\t4,\n"
"\t\t\t\t\t\"Column\":\t{\n"
"\t\t\t\t\t\t\"Type\":\t9,\n"
"\t\t\t\t\t\t\"ColId\":\t1,\n"
"\t\t\t\t\t\t\"Bytes\":\t8\n"
"\t\t\t\t\t}\n"
"\t\t\t\t}\n"
"\t\t\t}, {\n"
"\t\t\t\t\"Base\":\t{\n"
"\t\t\t\t\t\"Schema\":\t{\n"
"\t\t\t\t\t\t\"Type\":\t4,\n"
"\t\t\t\t\t\t\"ColId\":\t5001,\n"
"\t\t\t\t\t\t\"Bytes\":\t4\n"
"\t\t\t\t\t},\n"
"\t\t\t\t\t\"Columns\":\t[{\n"
"\t\t\t\t\t\t\t\"TableId\":\t1,\n"
"\t\t\t\t\t\t\t\"Flag\":\t0,\n"
"\t\t\t\t\t\t\t\"Info\":\t{\n"
"\t\t\t\t\t\t\t\t\"ColId\":\t2,\n"
"\t\t\t\t\t\t\t\t\"Type\":\t4,\n"
"\t\t\t\t\t\t\t\t\"Bytes\":\t4\n"
"\t\t\t\t\t\t\t}\n"
"\t\t\t\t\t\t}],\n"
"\t\t\t\t\t\"InterBytes\":\t0\n"
"\t\t\t\t},\n"
"\t\t\t\t\"Expr\":\t{\n"
"\t\t\t\t\t\"Type\":\t4,\n"
"\t\t\t\t\t\"Column\":\t{\n"
"\t\t\t\t\t\t\"Type\":\t4,\n"
"\t\t\t\t\t\t\"ColId\":\t2,\n"
"\t\t\t\t\t\t\"Bytes\":\t4\n"
"\t\t\t\t\t}\n"
"\t\t\t\t}\n"
"\t\t\t}],\n"
"\t\t\"InputSchema\":\t[{\n"
"\t\t\t\t\"Type\":\t9,\n"
"\t\t\t\t\"ColId\":\t5000,\n"
"\t\t\t\t\"Bytes\":\t8\n"
"\t\t\t}, {\n"
"\t\t\t\t\"Type\":\t4,\n"
"\t\t\t\t\"ColId\":\t5001,\n"
"\t\t\t\t\"Bytes\":\t4\n"
"\t\t\t}],\n"
"\t\t\"TableScan\":\t{\n"
"\t\t\t\"TableId\":\t1,\n"
"\t\t\t\"TableType\":\t2,\n"
"\t\t\t\"Flag\":\t0,\n"
"\t\t\t\"Window\":\t{\n"
"\t\t\t\t\"StartKey\":\t-9.2233720368547758e+18,\n"
"\t\t\t\t\"EndKey\":\t9.2233720368547758e+18\n"
"\t\t\t}\n"
"\t\t}\n"
"\t},\n"
"\t\"DataSink\":\t{\n"
"\t\t\"Name\":\t\"Dispatch\",\n"
"\t\t\"Dispatch\":\t{\n"
"\t\t}\n"
"\t}\n"
"}";
SExecTaskInfo* pTaskInfo = nullptr;
int32_t code = qCreateExecTask((void*) 1, 2, NULL, (void**) &pTaskInfo);
}
\ No newline at end of file
source/libs/parser/src/parser.c
浏览文件 @ 369b5559
...@@ -70,6 +70,9 @@ int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) { ...@@ -70,6 +70,9 @@ int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) {
int32_t code = qParserValidateSqlNode(&pCxt->ctx, &info, pQueryInfo, pCxt->pMsg, pCxt->msgLen); int32_t code = qParserValidateSqlNode(&pCxt->ctx, &info, pQueryInfo, pCxt->pMsg, pCxt->msgLen);
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
*pQuery = (SQueryNode*)pQueryInfo; *pQuery = (SQueryNode*)pQueryInfo;
} else {
terrno = code;
return code;
} }
} }
......
source/libs/planner/inc/plannerInt.h
浏览文件 @ 369b5559
...@@ -62,7 +62,7 @@ typedef struct SQueryPlanNode { ...@@ -62,7 +62,7 @@ typedef struct SQueryPlanNode {
SSchema *pSchema; // the schema of the input SSDatablock SSchema *pSchema; // the schema of the input SSDatablock
int32_t numOfCols; // number of input columns int32_t numOfCols; // number of input columns
SArray *pExpr; // the query functions or sql aggregations SArray *pExpr; // the query functions or sql aggregations
int32_t numOfExpr; // number of result columns, which is also the number of pExprs int32_t numOfExpr; // number of result columns, which is also the number of pExprs
void *pExtInfo; // additional information void *pExtInfo; // additional information
// children operator to generated result for current node to process // children operator to generated result for current node to process
// in case of join, multiple prev nodes exist. // in case of join, multiple prev nodes exist.
......
source/libs/planner/src/physicalPlan.c
浏览文件 @ 369b5559
...@@ -88,16 +88,20 @@ static bool copySchema(SDataBlockSchema* dst, const SDataBlockSchema* src) { ...@@ -88,16 +88,20 @@ static bool copySchema(SDataBlockSchema* dst, const SDataBlockSchema* src) {
} }
static bool toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) { static bool toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) {
dataBlockSchema->numOfCols = pPlanNode->numOfCols; dataBlockSchema->numOfCols = pPlanNode->numOfExpr;
dataBlockSchema->pSchema = malloc(sizeof(SSlotSchema) * pPlanNode->numOfCols); dataBlockSchema->pSchema = malloc(sizeof(SSlotSchema) * pPlanNode->numOfExpr);
if (NULL == dataBlockSchema->pSchema) { if (NULL == dataBlockSchema->pSchema) {
return false; return false;
} }
memcpy(dataBlockSchema->pSchema, pPlanNode->pSchema, sizeof(SSlotSchema) * pPlanNode->numOfCols);
dataBlockSchema->resultRowSize = 0; dataBlockSchema->resultRowSize = 0;
for (int32_t i = 0; i < dataBlockSchema->numOfCols; ++i) { for (int32_t i = 0; i < pPlanNode->numOfExpr; ++i) {
SExprInfo* pExprInfo = taosArrayGetP(pPlanNode->pExpr, i);
memcpy(&dataBlockSchema->pSchema[i], &pExprInfo->base.resSchema, sizeof(SSlotSchema));
dataBlockSchema->resultRowSize += dataBlockSchema->pSchema[i].bytes; dataBlockSchema->resultRowSize += dataBlockSchema->pSchema[i].bytes;
} }
return true; return true;
} }
...@@ -284,7 +288,6 @@ static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTabl ...@@ -284,7 +288,6 @@ static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTabl
return createUserTableScanNode(pPlanNode, pTable, OP_TableScan); return createUserTableScanNode(pPlanNode, pTable, OP_TableScan);
} }
static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) {
SQueryTableInfo* pTable = (SQueryTableInfo*)pPlanNode->pExtInfo; SQueryTableInfo* pTable = (SQueryTableInfo*)pPlanNode->pExtInfo;
...@@ -303,8 +306,6 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { ...@@ -303,8 +306,6 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) {
case QNODE_TABLESCAN: case QNODE_TABLESCAN:
node = createTableScanNode(pCxt, pPlanNode); node = createTableScanNode(pCxt, pPlanNode);
break; break;
case QNODE_PROJECT:
// node = create
case QNODE_MODIFY: case QNODE_MODIFY:
// Insert is not an operator in a physical plan. // Insert is not an operator in a physical plan.
break; break;
......
source/libs/planner/src/physicalPlanJson.c
浏览文件 @ 369b5559
...@@ -154,9 +154,13 @@ static bool fromRawArrayWithAlloc(const cJSON* json, const char* name, FFromJson ...@@ -154,9 +154,13 @@ static bool fromRawArrayWithAlloc(const cJSON* json, const char* name, FFromJson
return fromItem(jArray, func, *array, itemSize, *size); return fromItem(jArray, func, *array, itemSize, *size);
} }
static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void* array, int32_t itemSize, int32_t* size) { static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void** array, int32_t itemSize, int32_t* size) {
const cJSON* jArray = getArray(json, name, size); const cJSON* jArray = getArray(json, name, size);
return fromItem(jArray, func, array, itemSize, *size); if (*array == NULL) {
*array = calloc(*size, itemSize);
}
return fromItem(jArray, func, *array, itemSize, *size);
} }
static char* getString(const cJSON* json, const char* name) { static char* getString(const cJSON* json, const char* name) {
...@@ -218,7 +222,8 @@ static bool dataBlockSchemaFromJson(const cJSON* json, void* obj) { ...@@ -218,7 +222,8 @@ static bool dataBlockSchemaFromJson(const cJSON* json, void* obj) {
SDataBlockSchema* schema = (SDataBlockSchema*)obj; SDataBlockSchema* schema = (SDataBlockSchema*)obj;
schema->resultRowSize = getNumber(json, jkDataBlockSchemaResultRowSize); schema->resultRowSize = getNumber(json, jkDataBlockSchemaResultRowSize);
schema->precision = getNumber(json, jkDataBlockSchemaPrecision); schema->precision = getNumber(json, jkDataBlockSchemaPrecision);
return fromRawArray(json, jkDataBlockSchemaSlotSchema, schemaFromJson, schema->pSchema, sizeof(SSlotSchema), &schema->numOfCols);
return fromRawArray(json, jkDataBlockSchemaSlotSchema, schemaFromJson, (void**) &(schema->pSchema), sizeof(SSlotSchema), &schema->numOfCols);
} }
static const char* jkColumnFilterInfoLowerRelOptr = "LowerRelOptr"; static const char* jkColumnFilterInfoLowerRelOptr = "LowerRelOptr";
...@@ -920,6 +925,8 @@ int32_t subPlanToString(const SSubplan* subplan, char** str, int32_t* len) { ...@@ -920,6 +925,8 @@ int32_t subPlanToString(const SSubplan* subplan, char** str, int32_t* len) {
} }
*str = cJSON_Print(json); *str = cJSON_Print(json);
printf("%s\n", *str);
*len = strlen(*str) + 1; *len = strlen(*str) + 1;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
......
source/libs/planner/src/planner.c
浏览文件 @ 369b5559
...@@ -57,34 +57,33 @@ void qDestroyQueryDag(struct SQueryDag* pDag) { ...@@ -57,34 +57,33 @@ void qDestroyQueryDag(struct SQueryDag* pDag) {
} }
int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, uint64_t requestId) { int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, uint64_t requestId) {
SQueryPlanNode* logicPlan; SQueryPlanNode* pLogicPlan;
int32_t code = createQueryPlan(pNode, &logicPlan); int32_t code = createQueryPlan(pNode, &pLogicPlan);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
destroyQueryPlan(logicPlan); destroyQueryPlan(pLogicPlan);
return code; return code;
} }
// if (pLogicPlan->info.type != QNODE_MODIFY) {
if (logicPlan->info.type != QNODE_MODIFY) { char* str = NULL;
// char* str = NULL; queryPlanToString(pLogicPlan, &str);
// queryPlanToString(logicPlan, &str); printf("%s\n", str);
// printf("%s\n", str);
} }
code = optimizeQueryPlan(logicPlan); code = optimizeQueryPlan(pLogicPlan);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
destroyQueryPlan(logicPlan); destroyQueryPlan(pLogicPlan);
return code; return code;
} }
code = createDag(logicPlan, NULL, pDag, requestId); code = createDag(pLogicPlan, NULL, pDag, requestId);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
destroyQueryPlan(logicPlan); destroyQueryPlan(pLogicPlan);
qDestroyQueryDag(*pDag); qDestroyQueryDag(*pDag);
return code; return code;
} }
destroyQueryPlan(logicPlan); destroyQueryPlan(pLogicPlan);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
......
source/libs/planner/test/phyPlanTests.cpp
浏览文件 @ 369b5559
...@@ -62,6 +62,9 @@ protected: ...@@ -62,6 +62,9 @@ protected:
} }
SQueryDag* dag = nullptr; SQueryDag* dag = nullptr;
uint64_t requestId = 20; uint64_t requestId = 20;
SSchema *schema = NULL;
uint32_t numOfOutput = 0;
code = qCreateQueryDag(query, &dag, requestId); code = qCreateQueryDag(query, &dag, requestId);
dag_.reset(dag); dag_.reset(dag);
return code; return code;
......
source/libs/qworker/CMakeLists.txt
浏览文件 @ 369b5559
aux_source_directory(src QWORKER_SRC) aux_source_directory(src QWORKER_SRC)
add_library(qworker ${QWORKER_SRC}) #add_library(qworker ${QWORKER_SRC})
#target_include_directories(
# qworker
# PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/qworker"
# PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
#)
#
#target_link_libraries(
# qworker
# PRIVATE os util transport planner qcom executor
#)
add_library(qworker STATIC ${QWORKER_SRC})
target_include_directories( target_include_directories(
qworker qworker
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/qworker" PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/qworker"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
) )
target_link_libraries( #set_target_properties(qworker PROPERTIES
qworker # IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/libqworker.a"
PRIVATE os util transport planner qcom # INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/libs/qworker"
) # )
target_link_libraries(qworker
PRIVATE os util transport planner qcom executor
)
if(${BUILD_TEST}) if(${BUILD_TEST})
ADD_SUBDIRECTORY(test) ADD_SUBDIRECTORY(test)
......
source/libs/qworker/inc/qworkerInt.h
浏览文件 @ 369b5559
...@@ -67,15 +67,17 @@ typedef struct SQWTaskStatus { ...@@ -67,15 +67,17 @@ typedef struct SQWTaskStatus {
bool drop; bool drop;
} SQWTaskStatus; } SQWTaskStatus;
typedef struct SQWorkerResCache { typedef struct SQWorkerTaskHandlesCache {
SRWLatch lock; SRWLatch lock;
void *data; bool needRsp;
} SQWorkerResCache; qTaskInfo_t taskHandle;
DataSinkHandle sinkHandle;
} SQWorkerTaskHandlesCache;
typedef struct SQWSchStatus { typedef struct SQWSchStatus {
int32_t lastAccessTs; // timestamp in second int32_t lastAccessTs; // timestamp in second
SRWLatch tasksLock; SRWLatch tasksLock;
SHashObj *tasksHash; // key:queryId+taskId, value: SQWorkerTaskStatus SHashObj *tasksHash; // key:queryId+taskId, value: SQWTaskStatus
} SQWSchStatus; } SQWSchStatus;
// Qnode/Vnode level task management // Qnode/Vnode level task management
...@@ -83,7 +85,7 @@ typedef struct SQWorkerMgmt { ...@@ -83,7 +85,7 @@ typedef struct SQWorkerMgmt {
SQWorkerCfg cfg; SQWorkerCfg cfg;
SRWLatch schLock; SRWLatch schLock;
SRWLatch resLock; SRWLatch resLock;
SHashObj *schHash; //key: schedulerId, value: SQWorkerSchStatus SHashObj *schHash; //key: schedulerId, value: SQWSchStatus
SHashObj *resHash; //key: queryId+taskId, value: SQWorkerResCache SHashObj *resHash; //key: queryId+taskId, value: SQWorkerResCache
} SQWorkerMgmt; } SQWorkerMgmt;
......
source/libs/qworker/src/qworker.c
浏览文件 @ 369b5559
#include "qworker.h" #include "qworker.h"
#include "tname.h" #include <common.h>
#include "executor.h"
#include "planner.h" #include "planner.h"
#include "query.h" #include "query.h"
#include "qworkerInt.h" #include "qworkerInt.h"
#include "tmsg.h" #include "tmsg.h"
#include "tname.h"
#include "dataSinkMgt.h"
int32_t qwValidateStatus(int8_t oriStatus, int8_t newStatus) { int32_t qwValidateStatus(int8_t oriStatus, int8_t newStatus) {
int32_t code = 0; int32_t code = 0;
...@@ -89,15 +92,16 @@ int32_t qwUpdateTaskInfo(SQWTaskStatus *task, int8_t type, void *data) { ...@@ -89,15 +92,16 @@ int32_t qwUpdateTaskInfo(SQWTaskStatus *task, int8_t type, void *data) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t qwAddTaskResCache(SQWorkerMgmt *mgmt, uint64_t qId, uint64_t tId, void *data) { int32_t qwAddTaskHandlesToCache(SQWorkerMgmt *mgmt, uint64_t qId, uint64_t tId, qTaskInfo_t taskHandle, DataSinkHandle sinkHandle) {
char id[sizeof(qId) + sizeof(tId)] = {0}; char id[sizeof(qId) + sizeof(tId)] = {0};
QW_SET_QTID(id, qId, tId); QW_SET_QTID(id, qId, tId);
SQWorkerResCache resCache = {0}; SQWorkerTaskHandlesCache resCache = {0};
resCache.data = data; resCache.taskHandle = taskHandle;
resCache.sinkHandle = sinkHandle;
QW_LOCK(QW_WRITE, &mgmt->resLock); QW_LOCK(QW_WRITE, &mgmt->resLock);
if (0 != taosHashPut(mgmt->resHash, id, sizeof(id), &resCache, sizeof(SQWorkerResCache))) { if (0 != taosHashPut(mgmt->resHash, id, sizeof(id), &resCache, sizeof(SQWorkerTaskHandlesCache))) {
QW_UNLOCK(QW_WRITE, &mgmt->resLock); QW_UNLOCK(QW_WRITE, &mgmt->resLock);
qError("taosHashPut queryId[%"PRIx64"] taskId[%"PRIx64"] to resHash failed", qId, tId); qError("taosHashPut queryId[%"PRIx64"] taskId[%"PRIx64"] to resHash failed", qId, tId);
return TSDB_CODE_QRY_APP_ERROR; return TSDB_CODE_QRY_APP_ERROR;
...@@ -246,13 +250,13 @@ static int32_t qwAddTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ ...@@ -246,13 +250,13 @@ static int32_t qwAddTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_
QW_RET(code); QW_RET(code);
} }
static FORCE_INLINE int32_t qwAcquireTaskResCache(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t queryId, uint64_t taskId, SQWorkerResCache **res) { static FORCE_INLINE int32_t qwAcquireTaskHandles(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t queryId, uint64_t taskId, SQWorkerTaskHandlesCache **handles) {
char id[sizeof(queryId) + sizeof(taskId)] = {0}; char id[sizeof(queryId) + sizeof(taskId)] = {0};
QW_SET_QTID(id, queryId, taskId); QW_SET_QTID(id, queryId, taskId);
QW_LOCK(rwType, &mgmt->resLock); QW_LOCK(rwType, &mgmt->resLock);
*res = taosHashGet(mgmt->resHash, id, sizeof(id)); *handles = taosHashGet(mgmt->resHash, id, sizeof(id));
if (NULL == (*res)) { if (NULL == (*handles)) {
QW_UNLOCK(rwType, &mgmt->resLock); QW_UNLOCK(rwType, &mgmt->resLock);
return TSDB_CODE_QRY_RES_CACHE_NOT_EXIST; return TSDB_CODE_QRY_RES_CACHE_NOT_EXIST;
} }
...@@ -602,19 +606,36 @@ int32_t qwBuildAndSendStatusRsp(SRpcMsg *pMsg, SSchedulerStatusRsp *sStatus) { ...@@ -602,19 +606,36 @@ int32_t qwBuildAndSendStatusRsp(SRpcMsg *pMsg, SSchedulerStatusRsp *sStatus) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t qwBuildAndSendFetchRsp(SRpcMsg *pMsg, void *data) { int32_t qwInitFetchRsp(int32_t length, SRetrieveTableRsp **rsp) {
SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); int32_t msgSize = sizeof(SRetrieveTableRsp) + length;
SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcMallocCont(msgSize);
if (NULL == pRsp) {
qError("rpcMallocCont %d failed", msgSize);
QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
memset(pRsp, 0, sizeof(SRetrieveTableRsp)); memset(pRsp, 0, sizeof(SRetrieveTableRsp));
//TODO fill msg *rsp = pRsp;
pRsp->completed = true;
return TSDB_CODE_SUCCESS;
}
int32_t qwBuildAndSendFetchRsp(SRpcMsg *pMsg, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code) {
if (NULL == pRsp) {
pRsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
memset(pRsp, 0, sizeof(SRetrieveTableRsp));
dataLength = 0;
}
SRpcMsg rpcRsp = { SRpcMsg rpcRsp = {
.handle = pMsg->handle, .handle = pMsg->handle,
.ahandle = pMsg->ahandle, .ahandle = pMsg->ahandle,
.pCont = pRsp, .pCont = pRsp,
.contLen = sizeof(*pRsp), .contLen = sizeof(*pRsp) + dataLength,
.code = 0, .code = code,
}; };
rpcSendResponse(&rpcRsp); rpcSendResponse(&rpcRsp);
...@@ -847,62 +868,6 @@ int32_t qwCheckTaskCancelDrop( SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryI ...@@ -847,62 +868,6 @@ int32_t qwCheckTaskCancelDrop( SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryI
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t qwHandleFetch(SQWorkerResCache *res, SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg) {
SQWSchStatus *sch = NULL;
SQWTaskStatus *task = NULL;
int32_t code = 0;
int32_t needRsp = true;
void *data = NULL;
QW_ERR_JRET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR));
QW_ERR_JRET(qwAcquireTask(QW_READ, sch, queryId, taskId, &task));
QW_LOCK(QW_READ, &task->lock);
if (task->cancel || task->drop) {
qError("task is already cancelled or dropped");
QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
}
if (task->status != JOB_TASK_STATUS_EXECUTING && task->status != JOB_TASK_STATUS_PARTIAL_SUCCEED) {
qError("invalid status %d for fetch", task->status);
QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
}
if (QW_GOT_RES_DATA(res->data)) {
data = res->data;
if (QW_LOW_RES_DATA(res->data)) {
if (task->status == JOB_TASK_STATUS_PARTIAL_SUCCEED) {
//TODO add query back to queue
}
}
} else {
if (task->status != JOB_TASK_STATUS_EXECUTING) {
qError("invalid status %d for fetch without res", task->status);
QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
}
//TODO SET FLAG FOR QUERY TO SEND RSP WHEN RES READY
needRsp = false;
}
_return:
if (task) {
QW_UNLOCK(QW_READ, &task->lock);
qwReleaseTask(QW_READ, sch);
}
if (sch) {
qwReleaseScheduler(QW_READ, mgmt);
}
if (needRsp) {
qwBuildAndSendFetchRsp(pMsg, res->data);
}
QW_RET(code);
}
int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int8_t status, int32_t errCode) { int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int8_t status, int32_t errCode) {
SQWSchStatus *sch = NULL; SQWSchStatus *sch = NULL;
...@@ -956,6 +921,113 @@ int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint6 ...@@ -956,6 +921,113 @@ int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint6
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg) {
SQWSchStatus *sch = NULL;
SQWTaskStatus *task = NULL;
int32_t code = 0;
int32_t needRsp = true;
void *data = NULL;
int32_t sinkStatus = 0;
int32_t dataLength = 0;
SRetrieveTableRsp *rsp = NULL;
bool queryEnd = false;
SQWorkerTaskHandlesCache *handles = NULL;
QW_ERR_JRET(qwAcquireTaskHandles(QW_READ, mgmt, queryId, taskId, &handles));
QW_ERR_JRET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR));
QW_ERR_JRET(qwAcquireTask(QW_READ, sch, queryId, taskId, &task));
QW_LOCK(QW_READ, &task->lock);
if (task->cancel || task->drop) {
qError("task is already cancelled or dropped");
QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
}
if (task->status != JOB_TASK_STATUS_EXECUTING && task->status != JOB_TASK_STATUS_PARTIAL_SUCCEED) {
qError("invalid status %d for fetch", task->status);
QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
}
dsGetDataLength(handles->sinkHandle, &dataLength, &queryEnd);
if (dataLength > 0) {
SOutputData output = {0};
QW_ERR_JRET(qwInitFetchRsp(dataLength, &rsp));
output.pData = rsp->data;
code = dsGetDataBlock(handles->sinkHandle, &output);
if (code) {
qError("dsGetDataBlock failed, code:%x", code);
QW_ERR_JRET(code);
}
rsp->useconds = htobe64(output.useconds);
rsp->completed = 0;
rsp->precision = output.precision;
rsp->compressed = output.compressed;
rsp->compLen = htonl(dataLength);
rsp->numOfRows = htonl(output.numOfRows);
if (DS_BUF_EMPTY == output.bufStatus && output.queryEnd) {
rsp->completed = 1;
}
if (output.needSchedule) {
//TODO
}
if ((!output.queryEnd) && DS_BUF_LOW == output.bufStatus) {
//TODO
//UPDATE STATUS TO EXECUTING
}
} else {
if (dataLength < 0) {
qError("invalid length from dsGetDataLength, length:%d", dataLength);
QW_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
}
if (queryEnd) {
QW_ERR_JRET(qwQueryPostProcess(mgmt, sId, queryId, taskId, JOB_TASK_STATUS_SUCCEED, code));
} else {
if (task->status != JOB_TASK_STATUS_EXECUTING) {
qError("invalid status %d for fetch without res", task->status);
QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
}
QW_LOCK(QW_WRITE, &handles->lock);
handles->needRsp = true;
QW_UNLOCK(QW_WRITE, &handles->lock);
needRsp = false;
}
}
_return:
if (task) {
QW_UNLOCK(QW_READ, &task->lock);
qwReleaseTask(QW_READ, sch);
}
if (sch) {
qwReleaseScheduler(QW_READ, mgmt);
}
if (needRsp) {
qwBuildAndSendFetchRsp(pMsg, rsp, dataLength, code);
}
if (handles) {
qwReleaseTaskResCache(QW_READ, mgmt);
}
QW_RET(code);
}
int32_t qWorkerInit(SQWorkerCfg *cfg, void **qWorkerMgmt) { int32_t qWorkerInit(SQWorkerCfg *cfg, void **qWorkerMgmt) {
SQWorkerMgmt *mgmt = calloc(1, sizeof(SQWorkerMgmt)); SQWorkerMgmt *mgmt = calloc(1, sizeof(SQWorkerMgmt));
if (NULL == mgmt) { if (NULL == mgmt) {
...@@ -1008,10 +1080,9 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { ...@@ -1008,10 +1080,9 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
msg->taskId = htobe64(msg->taskId); msg->taskId = htobe64(msg->taskId);
msg->contentLen = ntohl(msg->contentLen); msg->contentLen = ntohl(msg->contentLen);
bool queryDone = false;
bool queryRsped = false; bool queryRsped = false;
bool needStop = false; bool needStop = false;
SSubplan *plan = NULL; struct SSubplan *plan = NULL;
QW_ERR_JRET(qwCheckTaskCancelDrop(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, &needStop)); QW_ERR_JRET(qwCheckTaskCancelDrop(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, &needStop));
if (needStop) { if (needStop) {
...@@ -1025,11 +1096,10 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { ...@@ -1025,11 +1096,10 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
QW_ERR_JRET(code); QW_ERR_JRET(code);
} }
//TODO call executer to init subquery qTaskInfo_t pTaskInfo = NULL;
code = 0; // return error directly code = qCreateExecTask(node, 0, (struct SSubplan *)plan, &pTaskInfo);
//TODO call executer to init subquery
if (code) { if (code) {
qError("qCreateExecTask failed, code:%x", code);
QW_ERR_JRET(code); QW_ERR_JRET(code);
} else { } else {
QW_ERR_JRET(qwAddTask(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, JOB_TASK_STATUS_EXECUTING, QW_EXIST_RET_ERR, NULL, NULL)); QW_ERR_JRET(qwAddTask(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, JOB_TASK_STATUS_EXECUTING, QW_EXIST_RET_ERR, NULL, NULL));
...@@ -1038,18 +1108,15 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { ...@@ -1038,18 +1108,15 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
QW_ERR_JRET(qwBuildAndSendQueryRsp(pMsg, TSDB_CODE_SUCCESS)); QW_ERR_JRET(qwBuildAndSendQueryRsp(pMsg, TSDB_CODE_SUCCESS));
queryRsped = true; queryRsped = true;
//TODO call executer to execute subquery DataSinkHandle sinkHandle = NULL;
code = 0; code = qExecTask(pTaskInfo, &sinkHandle);
void *data = NULL;
queryDone = false;
//TODO call executer to execute subquery
if (code) { if (code) {
qError("qExecTask failed, code:%x", code);
QW_ERR_JRET(code); QW_ERR_JRET(code);
} else { } else {
QW_ERR_JRET(qwAddTaskResCache(qWorkerMgmt, msg->queryId, msg->taskId, data)); QW_ERR_JRET(qwAddTaskHandlesToCache(qWorkerMgmt, msg->queryId, msg->taskId, pTaskInfo, sinkHandle));
QW_ERR_JRET(qwUpdateTaskStatus(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, JOB_TASK_STATUS_PARTIAL_SUCCEED)); QW_ERR_JRET(qwUpdateTaskStatus(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, JOB_TASK_STATUS_PARTIAL_SUCCEED));
} }
...@@ -1064,8 +1131,6 @@ _return: ...@@ -1064,8 +1131,6 @@ _return:
int8_t status = 0; int8_t status = 0;
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
status = JOB_TASK_STATUS_FAILED; status = JOB_TASK_STATUS_FAILED;
} else if (queryDone) {
status = JOB_TASK_STATUS_SUCCEED;
} else { } else {
status = JOB_TASK_STATUS_PARTIAL_SUCCEED; status = JOB_TASK_STATUS_PARTIAL_SUCCEED;
} }
...@@ -1075,6 +1140,49 @@ _return: ...@@ -1075,6 +1140,49 @@ _return:
QW_RET(code); QW_RET(code);
} }
int32_t qWorkerProcessQueryContinueMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
int32_t code = 0;
int8_t status = 0;
bool queryDone = false;
uint64_t sId, qId, tId;
//TODO call executer to continue execute subquery
code = 0;
void *data = NULL;
queryDone = false;
//TODO call executer to continue execute subquery
if (TSDB_CODE_SUCCESS != code) {
status = JOB_TASK_STATUS_FAILED;
} else if (queryDone) {
status = JOB_TASK_STATUS_SUCCEED;
} else {
status = JOB_TASK_STATUS_PARTIAL_SUCCEED;
}
code = qwQueryPostProcess(qWorkerMgmt, sId, qId, tId, status, code);
QW_RET(code);
}
int32_t qWorkerProcessSinkDataMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){
if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) {
return TSDB_CODE_QRY_INVALID_INPUT;
}
SSinkDataReq *msg = pMsg->pCont;
if (NULL == msg || pMsg->contLen < sizeof(*msg)) {
qError("invalid sink data msg");
QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
//TODO
return TSDB_CODE_SUCCESS;
}
int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){ int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){
if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) { if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) {
return TSDB_CODE_QRY_INVALID_INPUT; return TSDB_CODE_QRY_INVALID_INPUT;
...@@ -1137,17 +1245,10 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { ...@@ -1137,17 +1245,10 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
QW_ERR_RET(qwUpdateSchLastAccess(qWorkerMgmt, msg->sId)); QW_ERR_RET(qwUpdateSchLastAccess(qWorkerMgmt, msg->sId));
void *data = NULL; void *data = NULL;
SQWorkerResCache *res = NULL;
int32_t code = 0; int32_t code = 0;
QW_ERR_RET(qwAcquireTaskResCache(QW_READ, qWorkerMgmt, msg->queryId, msg->taskId, &res)); QW_ERR_RET(qwHandleFetch(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, pMsg));
QW_ERR_JRET(qwHandleFetch(res, qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, pMsg));
_return:
qwReleaseTaskResCache(QW_READ, qWorkerMgmt);
QW_RET(code); QW_RET(code);
} }
...@@ -1220,31 +1321,6 @@ int32_t qWorkerProcessShowFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) ...@@ -1220,31 +1321,6 @@ int32_t qWorkerProcessShowFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg)
QW_ERR_RET(qwBuildAndSendShowFetchRsp(pMsg, pFetchReq)); QW_ERR_RET(qwBuildAndSendShowFetchRsp(pMsg, pFetchReq));
} }
int32_t qWorkerContinueQuery(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
int32_t code = 0;
int8_t status = 0;
bool queryDone = false;
uint64_t sId, qId, tId;
//TODO call executer to continue execute subquery
code = 0;
void *data = NULL;
queryDone = false;
//TODO call executer to continue execute subquery
if (TSDB_CODE_SUCCESS != code) {
status = JOB_TASK_STATUS_FAILED;
} else if (queryDone) {
status = JOB_TASK_STATUS_SUCCEED;
} else {
status = JOB_TASK_STATUS_PARTIAL_SUCCEED;
}
code = qwQueryPostProcess(qWorkerMgmt, sId, qId, tId, status, code);
QW_RET(code);
}
void qWorkerDestroy(void **qWorkerMgmt) { void qWorkerDestroy(void **qWorkerMgmt) {
if (NULL == qWorkerMgmt || NULL == *qWorkerMgmt) { if (NULL == qWorkerMgmt || NULL == *qWorkerMgmt) {
return; return;
......
source/libs/scheduler/inc/schedulerInt.h
浏览文件 @ 369b5559
...@@ -58,7 +58,6 @@ typedef struct SSchLevel { ...@@ -58,7 +58,6 @@ typedef struct SSchLevel {
SArray *subTasks; // Element is SQueryTask SArray *subTasks; // Element is SQueryTask
} SSchLevel; } SSchLevel;
typedef struct SSchTask { typedef struct SSchTask {
uint64_t taskId; // task id uint64_t taskId; // task id
SRWLatch lock; // task lock SRWLatch lock; // task lock
......
source/libs/scheduler/src/scheduler.c
浏览文件 @ 369b5559
...@@ -68,7 +68,7 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m ...@@ -68,7 +68,7 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
} }
if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXECUTING) { if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXECUTING && SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_PARTIAL_SUCCEED) {
SCH_TASK_ELOG("rsp msg conflicted with task status, status:%d, rspType:%d", SCH_GET_TASK_STATUS(pTask), msgType); SCH_TASK_ELOG("rsp msg conflicted with task status, status:%d, rspType:%d", SCH_GET_TASK_STATUS(pTask), msgType);
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
} }
...@@ -87,68 +87,72 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m ...@@ -87,68 +87,72 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m
int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) {
int32_t code = 0; int32_t code = 0;
int8_t oriStatus = SCH_GET_JOB_STATUS(pJob); int8_t oriStatus = 0;
/* while (true) {
if (oriStatus == newStatus) { oriStatus = SCH_GET_JOB_STATUS(pJob);
SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
} if (oriStatus == newStatus) {
SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
switch (oriStatus) { }
case JOB_TASK_STATUS_NULL:
if (newStatus != JOB_TASK_STATUS_EXECUTING switch (oriStatus) {
&& newStatus != JOB_TASK_STATUS_FAILED case JOB_TASK_STATUS_NULL:
&& newStatus != JOB_TASK_STATUS_NOT_START) { if (newStatus != JOB_TASK_STATUS_NOT_START) {
SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
} }
break; break;
case JOB_TASK_STATUS_NOT_START: case JOB_TASK_STATUS_NOT_START:
if (newStatus != JOB_TASK_STATUS_CANCELLED) { if (newStatus != JOB_TASK_STATUS_EXECUTING) {
SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
} }
break; break;
case JOB_TASK_STATUS_EXECUTING: case JOB_TASK_STATUS_EXECUTING:
if (newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED if (newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED
&& newStatus != JOB_TASK_STATUS_FAILED && newStatus != JOB_TASK_STATUS_FAILED
&& newStatus != JOB_TASK_STATUS_CANCELLING && newStatus != JOB_TASK_STATUS_CANCELLING
&& newStatus != JOB_TASK_STATUS_CANCELLED && newStatus != JOB_TASK_STATUS_CANCELLED
&& newStatus != JOB_TASK_STATUS_DROPPING) { && newStatus != JOB_TASK_STATUS_DROPPING) {
SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
} }
break; break;
case JOB_TASK_STATUS_PARTIAL_SUCCEED: case JOB_TASK_STATUS_PARTIAL_SUCCEED:
if (newStatus != JOB_TASK_STATUS_EXECUTING if (newStatus != JOB_TASK_STATUS_FAILED
&& newStatus != JOB_TASK_STATUS_SUCCEED && newStatus != JOB_TASK_STATUS_SUCCEED
&& newStatus != JOB_TASK_STATUS_CANCELLED) { && newStatus != JOB_TASK_STATUS_DROPPING) {
SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
}
break;
case JOB_TASK_STATUS_SUCCEED:
case JOB_TASK_STATUS_FAILED:
case JOB_TASK_STATUS_CANCELLING:
if (newStatus != JOB_TASK_STATUS_DROPPING) {
SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
}
break;
case JOB_TASK_STATUS_CANCELLED:
case JOB_TASK_STATUS_DROPPING:
SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
} break;
break; default:
case JOB_TASK_STATUS_SUCCEED: SCH_JOB_ELOG("invalid job status:%d", oriStatus);
case JOB_TASK_STATUS_FAILED:
case JOB_TASK_STATUS_CANCELLING:
if (newStatus != JOB_TASK_STATUS_CANCELLED) {
SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
} }
break;
case JOB_TASK_STATUS_CANCELLED:
case JOB_TASK_STATUS_DROPPING:
SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
break;
default:
qError("invalid task status:%d", oriStatus);
return TSDB_CODE_QRY_APP_ERROR;
}
*/
SCH_SET_JOB_STATUS(pJob, newStatus); if (oriStatus != atomic_val_compare_exchange_8(&pJob->status, oriStatus, newStatus)) {
continue;
}
SCH_JOB_DLOG("status updated from %d to %d", oriStatus, newStatus); SCH_JOB_DLOG("job status updated from %d to %d", oriStatus, newStatus);
break;
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
...@@ -265,7 +269,6 @@ int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *ad ...@@ -265,7 +269,6 @@ int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *ad
int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) {
int32_t code = 0; int32_t code = 0;
pJob->queryId = pDag->queryId; pJob->queryId = pDag->queryId;
if (pDag->numOfSubplans <= 0) { if (pDag->numOfSubplans <= 0) {
...@@ -363,7 +366,6 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { ...@@ -363,7 +366,6 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) {
SCH_ERR_JRET(schBuildTaskRalation(pJob, planToTask)); SCH_ERR_JRET(schBuildTaskRalation(pJob, planToTask));
_return: _return:
if (planToTask) { if (planToTask) {
taosHashCleanup(planToTask); taosHashCleanup(planToTask);
} }
...@@ -495,6 +497,33 @@ int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) { ...@@ -495,6 +497,33 @@ int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) {
if (0 != taosHashRemove(pJob->succTasks, &pTask->taskId, sizeof(pTask->taskId))) {
SCH_TASK_WLOG("remove task from succTask list failed, may not exist, status:%d", SCH_GET_TASK_STATUS(pTask));
}
int32_t code = taosHashPut(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId), &pTask, POINTER_BYTES);
if (0 != code) {
if (HASH_NODE_EXIST(code)) {
*moved = true;
SCH_TASK_ELOG("task already in execTask list, status:%d", SCH_GET_TASK_STATUS(pTask));
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
}
SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:%d", errno);
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
*moved = true;
SCH_TASK_DLOG("task moved to execTask list, numOfTasks:%d", taosHashGetSize(pJob->execTasks));
return TSDB_CODE_SUCCESS;
}
int32_t schTaskCheckAndSetRetry(SSchJob *job, SSchTask *task, int32_t errCode, bool *needRetry) { int32_t schTaskCheckAndSetRetry(SSchJob *job, SSchTask *task, int32_t errCode, bool *needRetry) {
// TODO set retry or not based on task type/errCode/retry times/job status/available eps... // TODO set retry or not based on task type/errCode/retry times/job status/available eps...
// TODO if needRetry, set task retry info // TODO if needRetry, set task retry info
...@@ -507,6 +536,28 @@ int32_t schTaskCheckAndSetRetry(SSchJob *job, SSchTask *task, int32_t errCode, b ...@@ -507,6 +536,28 @@ int32_t schTaskCheckAndSetRetry(SSchJob *job, SSchTask *task, int32_t errCode, b
} }
// Note: no more error processing, handled in function internal
int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) {
// if already FAILED, no more processing
SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_FAILED));
if (errCode) {
atomic_store_32(&pJob->errCode, errCode);
}
if (atomic_load_8(&pJob->userFetch) || ((!SCH_JOB_NEED_FETCH(&pJob->attr)) && pJob->attr.syncSchedule)) {
tsem_post(&pJob->rspSem);
}
SCH_ERR_RET(atomic_load_32(&pJob->errCode));
assert(0);
}
// Note: no more error processing, handled in function internal
int32_t schFetchFromRemote(SSchJob *pJob) { int32_t schFetchFromRemote(SSchJob *pJob) {
int32_t code = 0; int32_t code = 0;
...@@ -515,7 +566,13 @@ int32_t schFetchFromRemote(SSchJob *pJob) { ...@@ -515,7 +566,13 @@ int32_t schFetchFromRemote(SSchJob *pJob) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (atomic_load_ptr(&pJob->res)) void *res = atomic_load_ptr(&pJob->res);
if (res) {
atomic_val_compare_exchange_32(&pJob->remoteFetch, 1, 0);
SCH_JOB_DLOG("res already fetched, res:%p", res);
return TSDB_CODE_SUCCESS;
}
SCH_ERR_JRET(schBuildAndSendMsg(pJob, pJob->fetchTask, &pJob->resNode, TDMT_VND_FETCH)); SCH_ERR_JRET(schBuildAndSendMsg(pJob, pJob->fetchTask, &pJob->resNode, TDMT_VND_FETCH));
...@@ -525,25 +582,9 @@ _return: ...@@ -525,25 +582,9 @@ _return:
atomic_val_compare_exchange_32(&pJob->remoteFetch, 1, 0); atomic_val_compare_exchange_32(&pJob->remoteFetch, 1, 0);
return code; schProcessOnJobFailure(pJob, code);
}
return code;
// Note: no more error processing, handled in function internal
int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) {
SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_FAILED));
if (errCode) {
atomic_store_32(&pJob->errCode, errCode);
}
if (atomic_load_8(&pJob->userFetch) || ((!SCH_JOB_NEED_FETCH(&pJob->attr)) && pJob->attr.syncSchedule)) {
tsem_post(&pJob->rspSem);
}
SCH_ERR_RET(atomic_load_32(&pJob->errCode));
assert(0);
} }
...@@ -643,7 +684,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { ...@@ -643,7 +684,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) {
SCH_ERR_RET(code); SCH_ERR_RET(code);
} }
SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCCEED); SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_PARTIAL_SUCCEED);
SCH_ERR_JRET(schRecordTaskSucceedNode(pTask)); SCH_ERR_JRET(schRecordTaskSucceedNode(pTask));
...@@ -675,6 +716,11 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { ...@@ -675,6 +716,11 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) {
} }
pJob->fetchTask = pTask; pJob->fetchTask = pTask;
code = schMoveTaskToExecList(pJob, pTask, &moved);
if (code && moved) {
SCH_ERR_RET(code);
}
SCH_ERR_RET(schProcessOnJobPartialSuccess(pJob)); SCH_ERR_RET(schProcessOnJobPartialSuccess(pJob));
...@@ -813,7 +859,7 @@ int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, in ...@@ -813,7 +859,7 @@ int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, in
SSchJob **job = taosHashGet(schMgmt.jobs, &pParam->queryId, sizeof(pParam->queryId)); SSchJob **job = taosHashGet(schMgmt.jobs, &pParam->queryId, sizeof(pParam->queryId));
if (NULL == job || NULL == (*job)) { if (NULL == job || NULL == (*job)) {
qError("QID:%"PRIx64" taosHashGet queryId not exist", pParam->queryId); qError("QID:%"PRIx64" taosHashGet queryId not exist, may be dropped", pParam->queryId);
SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR);
} }
...@@ -822,7 +868,10 @@ int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, in ...@@ -822,7 +868,10 @@ int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, in
atomic_add_fetch_32(&pJob->ref, 1); atomic_add_fetch_32(&pJob->ref, 1);
int32_t s = taosHashGetSize(pJob->execTasks); int32_t s = taosHashGetSize(pJob->execTasks);
assert(s != 0); if (s <= 0) {
qError("QID:%"PRIx64",TID:%"PRIx64" no task in execTask list", pParam->queryId, pParam->taskId);
SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SSchTask **task = taosHashGet(pJob->execTasks, &pParam->taskId, sizeof(pParam->taskId)); SSchTask **task = taosHashGet(pJob->execTasks, &pParam->taskId, sizeof(pParam->taskId));
if (NULL == task || NULL == (*task)) { if (NULL == task || NULL == (*task)) {
...@@ -1147,7 +1196,7 @@ void schDropTaskOnExecutedNode(SSchJob *pJob, SSchTask *pTask) { ...@@ -1147,7 +1196,7 @@ void schDropTaskOnExecutedNode(SSchJob *pJob, SSchTask *pTask) {
int32_t size = (int32_t)taosArrayGetSize(pTask->execAddrs); int32_t size = (int32_t)taosArrayGetSize(pTask->execAddrs);
if (size <= 0) { if (size <= 0) {
SCH_TASK_DLOG("empty exec address, status:%d", SCH_GET_TASK_STATUS(pTask)); SCH_TASK_DLOG("task has no exec address, no need to drop it, status:%d", SCH_GET_TASK_STATUS(pTask));
return; return;
} }
...@@ -1157,6 +1206,8 @@ void schDropTaskOnExecutedNode(SSchJob *pJob, SSchTask *pTask) { ...@@ -1157,6 +1206,8 @@ void schDropTaskOnExecutedNode(SSchJob *pJob, SSchTask *pTask) {
schBuildAndSendMsg(pJob, pTask, addr, TDMT_VND_DROP_TASK); schBuildAndSendMsg(pJob, pTask, addr, TDMT_VND_DROP_TASK);
} }
SCH_TASK_DLOG("task has %d exec address", size);
} }
void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) {
...@@ -1178,7 +1229,7 @@ void schDropJobAllTasks(SSchJob *pJob) { ...@@ -1178,7 +1229,7 @@ void schDropJobAllTasks(SSchJob *pJob) {
schDropTaskInHashList(pJob, pJob->failTasks); schDropTaskInHashList(pJob, pJob->failTasks);
} }
int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag* pDag, void** job, bool syncSchedule) { int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** job, bool syncSchedule) {
if (nodeList && taosArrayGetSize(nodeList) <= 0) { if (nodeList && taosArrayGetSize(nodeList) <= 0) {
qInfo("QID:%"PRIx64" input nodeList is empty", pDag->queryId); qInfo("QID:%"PRIx64" input nodeList is empty", pDag->queryId);
} }
...@@ -1291,14 +1342,14 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { ...@@ -1291,14 +1342,14 @@ int32_t schedulerInit(SSchedulerCfg *cfg) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob, SQueryResult *pRes) { int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes) {
if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) {
SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
SSchJob *job = NULL; SSchJob *job = NULL;
SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, (void **)&job, true)); SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, &job, true));
*pJob = job; *pJob = job;
...@@ -1308,14 +1359,14 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void ...@@ -1308,14 +1359,14 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob) { int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob) {
if (NULL == transport || /*NULL == nodeList || */NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) {
SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
SSchJob *job = NULL; SSchJob *job = NULL;
SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, (void **)&job, false)); SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, &job, false));
*pJob = job; *pJob = job;
...@@ -1323,14 +1374,18 @@ int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, ...@@ -1323,14 +1374,18 @@ int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag,
} }
int32_t scheduleFetchRows(void *job, void **data) { int32_t scheduleFetchRows(SSchJob *pJob, void** pData) {
if (NULL == job || NULL == data) { if (NULL == pJob || NULL == pData) {
SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
SSchJob *pJob = job;
int32_t code = 0; int32_t code = 0;
int8_t status = SCH_GET_JOB_STATUS(pJob);
if (status == JOB_TASK_STATUS_DROPPING) {
SCH_JOB_ELOG("job is dropping, status:%d", status);
return TSDB_CODE_SCH_STATUS_ERROR;
}
atomic_add_fetch_32(&pJob->ref, 1); atomic_add_fetch_32(&pJob->ref, 1);
if (!SCH_JOB_NEED_FETCH(&pJob->attr)) { if (!SCH_JOB_NEED_FETCH(&pJob->attr)) {
...@@ -1345,14 +1400,12 @@ int32_t scheduleFetchRows(void *job, void **data) { ...@@ -1345,14 +1400,12 @@ int32_t scheduleFetchRows(void *job, void **data) {
SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
} }
int8_t status = SCH_GET_JOB_STATUS(pJob);
if (status == JOB_TASK_STATUS_FAILED) { if (status == JOB_TASK_STATUS_FAILED) {
*data = atomic_load_ptr(&pJob->res); *pData = atomic_load_ptr(&pJob->res);
atomic_store_ptr(&pJob->res, NULL); atomic_store_ptr(&pJob->res, NULL);
SCH_ERR_JRET(atomic_load_32(&pJob->errCode)); SCH_ERR_JRET(atomic_load_32(&pJob->errCode));
} else if (status == JOB_TASK_STATUS_SUCCEED) { } else if (status == JOB_TASK_STATUS_SUCCEED) {
*data = atomic_load_ptr(&pJob->res); *pData = atomic_load_ptr(&pJob->res);
atomic_store_ptr(&pJob->res, NULL); atomic_store_ptr(&pJob->res, NULL);
goto _return; goto _return;
} else if (status == JOB_TASK_STATUS_PARTIAL_SUCCEED) { } else if (status == JOB_TASK_STATUS_PARTIAL_SUCCEED) {
...@@ -1365,6 +1418,7 @@ int32_t scheduleFetchRows(void *job, void **data) { ...@@ -1365,6 +1418,7 @@ int32_t scheduleFetchRows(void *job, void **data) {
if (status == JOB_TASK_STATUS_FAILED) { if (status == JOB_TASK_STATUS_FAILED) {
code = atomic_load_32(&pJob->errCode); code = atomic_load_32(&pJob->errCode);
SCH_ERR_JRET(code);
} }
if (pJob->res && ((SRetrieveTableRsp *)pJob->res)->completed) { if (pJob->res && ((SRetrieveTableRsp *)pJob->res)->completed) {
...@@ -1372,9 +1426,9 @@ int32_t scheduleFetchRows(void *job, void **data) { ...@@ -1372,9 +1426,9 @@ int32_t scheduleFetchRows(void *job, void **data) {
} }
while (true) { while (true) {
*data = atomic_load_ptr(&pJob->res); *pData = atomic_load_ptr(&pJob->res);
if (*data != atomic_val_compare_exchange_ptr(&pJob->res, *data, NULL)) { if (*pData != atomic_val_compare_exchange_ptr(&pJob->res, *pData, NULL)) {
continue; continue;
} }
...@@ -1414,6 +1468,10 @@ void scheduleFreeJob(void *job) { ...@@ -1414,6 +1468,10 @@ void scheduleFreeJob(void *job) {
return; return;
} }
schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_DROPPING);
SCH_JOB_DLOG("job removed from list, no further ref, ref:%d", atomic_load_32(&pJob->ref));
while (true) { while (true) {
int32_t ref = atomic_load_32(&pJob->ref); int32_t ref = atomic_load_32(&pJob->ref);
if (0 == ref) { if (0 == ref) {
...@@ -1425,6 +1483,8 @@ void scheduleFreeJob(void *job) { ...@@ -1425,6 +1483,8 @@ void scheduleFreeJob(void *job) {
} }
} }
SCH_JOB_DLOG("job no ref now, status:%d", SCH_GET_JOB_STATUS(pJob));
if (pJob->status == JOB_TASK_STATUS_EXECUTING) { if (pJob->status == JOB_TASK_STATUS_EXECUTING) {
schCancelJob(pJob); schCancelJob(pJob);
} }
...@@ -1446,21 +1506,21 @@ void scheduleFreeJob(void *job) { ...@@ -1446,21 +1506,21 @@ void scheduleFreeJob(void *job) {
taosArrayDestroy(pLevel->subTasks); taosArrayDestroy(pLevel->subTasks);
} }
taosHashCleanup(pJob->execTasks); taosHashCleanup(pJob->execTasks);
taosHashCleanup(pJob->failTasks); taosHashCleanup(pJob->failTasks);
taosHashCleanup(pJob->succTasks); taosHashCleanup(pJob->succTasks);
taosArrayDestroy(pJob->levels);
tfree(pJob->res); taosArrayDestroy(pJob->levels);
tfree(pJob); tfree(pJob->res);
}
void schedulerDestroy(void) { tfree(pJob);
if (schMgmt.jobs) { }
taosHashCleanup(schMgmt.jobs); //TODO
schMgmt.jobs = NULL; void schedulerDestroy(void) {
} if (schMgmt.jobs) {
taosHashCleanup(schMgmt.jobs); //TODO
schMgmt.jobs = NULL;
} }
}
source/libs/scheduler/test/schedulerTests.cpp
浏览文件 @ 369b5559
...@@ -38,52 +38,73 @@ namespace { ...@@ -38,52 +38,73 @@ namespace {
extern "C" int32_t schHandleResponseMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode); extern "C" int32_t schHandleResponseMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode);
void schtInitLogFile() {
const char *defaultLogFileNamePrefix = "taoslog";
const int32_t maxLogFileNum = 10;
tsAsyncLog = 0;
qDebugFlag = 159;
char temp[128] = {0};
sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix);
if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) {
printf("failed to open log file in directory:%s\n", tsLogDir);
}
}
void schtBuildQueryDag(SQueryDag *dag) { void schtBuildQueryDag(SQueryDag *dag) {
uint64_t qId = 0x0000000000000001; uint64_t qId = 0x0000000000000001;
dag->queryId = qId; dag->queryId = qId;
dag->numOfSubplans = 2; dag->numOfSubplans = 2;
dag->pSubplans = taosArrayInit(dag->numOfSubplans, POINTER_BYTES); dag->pSubplans = taosArrayInit(dag->numOfSubplans, POINTER_BYTES);
SArray *scan = taosArrayInit(1, sizeof(SSubplan)); SArray *scan = taosArrayInit(1, POINTER_BYTES);
SArray *merge = taosArrayInit(1, sizeof(SSubplan)); SArray *merge = taosArrayInit(1, POINTER_BYTES);
SSubplan scanPlan = {0}; SSubplan *scanPlan = (SSubplan *)calloc(1, sizeof(SSubplan));
SSubplan mergePlan = {0}; SSubplan *mergePlan = (SSubplan *)calloc(1, sizeof(SSubplan));
scanPlan.id.queryId = qId; scanPlan->id.queryId = qId;
scanPlan.id.templateId = 0x0000000000000002; scanPlan->id.templateId = 0x0000000000000002;
scanPlan.id.subplanId = 0x0000000000000003; scanPlan->id.subplanId = 0x0000000000000003;
scanPlan.type = QUERY_TYPE_SCAN; scanPlan->type = QUERY_TYPE_SCAN;
scanPlan.execNode.numOfEps = 1; scanPlan->execNode.numOfEps = 1;
scanPlan.execNode.nodeId = 1; scanPlan->execNode.nodeId = 1;
scanPlan.execNode.inUse = 0; scanPlan->execNode.inUse = 0;
scanPlan.execNode.epAddr[0].port = 6030; scanPlan->execNode.epAddr[0].port = 6030;
strcpy(scanPlan.execNode.epAddr[0].fqdn, "ep0"); strcpy(scanPlan->execNode.epAddr[0].fqdn, "ep0");
scanPlan.pChildren = NULL; scanPlan->pChildren = NULL;
scanPlan.level = 1; scanPlan->level = 1;
scanPlan.pParents = taosArrayInit(1, POINTER_BYTES); scanPlan->pParents = taosArrayInit(1, POINTER_BYTES);
scanPlan.pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode)); scanPlan->pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode));
mergePlan.id.queryId = qId; mergePlan->id.queryId = qId;
mergePlan.id.templateId = 0x4444444444; mergePlan->id.templateId = 0x4444444444;
mergePlan.id.subplanId = 0x5555555555; mergePlan->id.subplanId = 0x5555555555;
mergePlan.type = QUERY_TYPE_MERGE; mergePlan->type = QUERY_TYPE_MERGE;
mergePlan.level = 0; mergePlan->level = 0;
mergePlan.execNode.numOfEps = 0; mergePlan->execNode.numOfEps = 0;
mergePlan.pChildren = taosArrayInit(1, POINTER_BYTES); mergePlan->pChildren = taosArrayInit(1, POINTER_BYTES);
mergePlan.pParents = NULL; mergePlan->pParents = NULL;
mergePlan.pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode)); mergePlan->pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode));
SSubplan *mergePointer = (SSubplan *)taosArrayPush(merge, &mergePlan); SSubplan *mergePointer = (SSubplan *)taosArrayPush(merge, &mergePlan);
SSubplan *scanPointer = (SSubplan *)taosArrayPush(scan, &scanPlan); SSubplan *scanPointer = (SSubplan *)taosArrayPush(scan, &scanPlan);
taosArrayPush(mergePointer->pChildren, &scanPointer); taosArrayPush(mergePlan->pChildren, &scanPlan);
taosArrayPush(scanPointer->pParents, &mergePointer); taosArrayPush(scanPlan->pParents, &mergePlan);
taosArrayPush(dag->pSubplans, &merge); taosArrayPush(dag->pSubplans, &merge);
taosArrayPush(dag->pSubplans, &scan); taosArrayPush(dag->pSubplans, &scan);
} }
void schtFreeQueryDag(SQueryDag *dag) {
}
void schtBuildInsertDag(SQueryDag *dag) { void schtBuildInsertDag(SQueryDag *dag) {
uint64_t qId = 0x0000000000000002; uint64_t qId = 0x0000000000000002;
...@@ -138,8 +159,8 @@ int32_t schtPlanToString(const SSubplan *subplan, char** str, int32_t* len) { ...@@ -138,8 +159,8 @@ int32_t schtPlanToString(const SSubplan *subplan, char** str, int32_t* len) {
return 0; return 0;
} }
int32_t schtExecNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep) { void schtExecNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep) {
return 0;
} }
...@@ -196,7 +217,7 @@ void *schtSendRsp(void *param) { ...@@ -196,7 +217,7 @@ void *schtSendRsp(void *param) {
return NULL; return NULL;
} }
void *pInsertJob = NULL; struct SSchJob *pInsertJob = NULL;
} }
...@@ -207,8 +228,11 @@ TEST(queryTest, normalCase) { ...@@ -207,8 +228,11 @@ TEST(queryTest, normalCase) {
char *dbname = "1.db1"; char *dbname = "1.db1";
char *tablename = "table1"; char *tablename = "table1";
SVgroupInfo vgInfo = {0}; SVgroupInfo vgInfo = {0};
void *pJob = NULL; SSchJob *pJob = NULL;
SQueryDag dag = {0}; SQueryDag dag = {0};
schtInitLogFile();
SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr));
SEpAddr qnodeAddr = {0}; SEpAddr qnodeAddr = {0};
...@@ -295,6 +319,8 @@ TEST(queryTest, normalCase) { ...@@ -295,6 +319,8 @@ TEST(queryTest, normalCase) {
ASSERT_EQ(data, (void*)NULL); ASSERT_EQ(data, (void*)NULL);
scheduleFreeJob(pJob); scheduleFreeJob(pJob);
schtFreeQueryDag(&dag);
} }
...@@ -308,6 +334,9 @@ TEST(insertTest, normalCase) { ...@@ -308,6 +334,9 @@ TEST(insertTest, normalCase) {
SVgroupInfo vgInfo = {0}; SVgroupInfo vgInfo = {0};
SQueryDag dag = {0}; SQueryDag dag = {0};
uint64_t numOfRows = 0; uint64_t numOfRows = 0;
schtInitLogFile();
SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr));
SEpAddr qnodeAddr = {0}; SEpAddr qnodeAddr = {0};
...@@ -336,7 +365,11 @@ TEST(insertTest, normalCase) { ...@@ -336,7 +365,11 @@ TEST(insertTest, normalCase) {
scheduleFreeJob(pInsertJob); scheduleFreeJob(pInsertJob);
} }
TEST(multiThread, forceFree) {
schtInitLogFile();
}
int main(int argc, char** argv) { int main(int argc, char** argv) {
......
source/os/src/osSysinfo.c
浏览文件 @ 369b5559
...@@ -1087,7 +1087,7 @@ int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { ...@@ -1087,7 +1087,7 @@ int32_t taosGetSystemUUID(char *uid, int32_t uidlen) {
return 0; return 0;
} }
return -1; return 0;
} }
char *taosGetCmdlineByPID(int pid) { char *taosGetCmdlineByPID(int pid) {
......
src/inc/tsdb.h
浏览文件 @ 369b5559
...@@ -274,7 +274,7 @@ TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STable ...@@ -274,7 +274,7 @@ TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STable
TsdbQueryHandleT tsdbQueryCacheLast(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pMemRef); TsdbQueryHandleT tsdbQueryCacheLast(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pMemRef);
bool isTsdbCacheLastRow(TsdbQueryHandleT* pQueryHandle); bool isTsdbCacheLastRow(TsdbQueryHandleT* pTsdbReadHandle);
/** /**
...@@ -308,19 +308,19 @@ int64_t tsdbGetNumOfRowsInMemTable(TsdbQueryHandleT* pHandle); ...@@ -308,19 +308,19 @@ int64_t tsdbGetNumOfRowsInMemTable(TsdbQueryHandleT* pHandle);
/** /**
* move to next block if exists * move to next block if exists
* *
* @param pQueryHandle * @param pTsdbReadHandle
* @return * @return
*/ */
bool tsdbNextDataBlock(TsdbQueryHandleT pQueryHandle); bool tsdbNextDataBlock(TsdbQueryHandleT pTsdbReadHandle);
/** /**
* Get current data block information * Get current data block information
* *
* @param pQueryHandle * @param pTsdbReadHandle
* @param pBlockInfo * @param pBlockInfo
* @return * @return
*/ */
void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle, SDataBlockInfo *pBlockInfo); void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo);
/** /**
* *
...@@ -332,7 +332,7 @@ void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle, SDataBlockInfo *p ...@@ -332,7 +332,7 @@ void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle, SDataBlockInfo *p
* @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0 * @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0
* @return * @return
*/ */
int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT *pQueryHandle, SDataStatis **pBlockStatis); int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT *pTsdbReadHandle, SDataStatis **pBlockStatis);
/** /**
* *
...@@ -340,11 +340,11 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT *pQueryHandle, SDataSta ...@@ -340,11 +340,11 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT *pQueryHandle, SDataSta
* the returned data block must be satisfied with the time window condition in any cases, * the returned data block must be satisfied with the time window condition in any cases,
* which means the SData data block is not actually the completed disk data blocks. * which means the SData data block is not actually the completed disk data blocks.
* *
* @param pQueryHandle query handle * @param pTsdbReadHandle query handle
* @param pColumnIdList required data columns id list * @param pColumnIdList required data columns id list
* @return * @return
*/ */
SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pQueryHandle, SArray *pColumnIdList); SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pTsdbReadHandle, SArray *pColumnIdList);
/** /**
* Get the qualified table id for a super table according to the tag query expression. * Get the qualified table id for a super table according to the tag query expression.
......
src/query/inc/qExecutor.h
浏览文件 @ 369b5559
...@@ -284,7 +284,7 @@ typedef struct SQueryRuntimeEnv { ...@@ -284,7 +284,7 @@ typedef struct SQueryRuntimeEnv {
uint32_t status; // query status uint32_t status; // query status
void* qinfo; void* qinfo;
uint8_t scanFlag; // denotes reversed scan of data or not uint8_t scanFlag; // denotes reversed scan of data or not
void* pQueryHandle; void* pTsdbReadHandle;
int32_t prevGroupId; // previous executed group id int32_t prevGroupId; // previous executed group id
bool enableGroupData; bool enableGroupData;
...@@ -418,7 +418,7 @@ typedef struct SQueryParam { ...@@ -418,7 +418,7 @@ typedef struct SQueryParam {
} SQueryParam; } SQueryParam;
typedef struct STableScanInfo { typedef struct STableScanInfo {
void *pQueryHandle; void *pTsdbReadHandle;
int32_t numOfBlocks; int32_t numOfBlocks;
int32_t numOfSkipped; int32_t numOfSkipped;
int32_t numOfBlockStatis; int32_t numOfBlockStatis;
......
src/query/src/qExecutor.c
浏览文件 @ 369b5559
...@@ -2326,8 +2326,8 @@ _clean: ...@@ -2326,8 +2326,8 @@ _clean:
static void doFreeQueryHandle(SQueryRuntimeEnv* pRuntimeEnv) { static void doFreeQueryHandle(SQueryRuntimeEnv* pRuntimeEnv) {
SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); tsdbCleanupQueryHandle(pRuntimeEnv->pTsdbReadHandle);
pRuntimeEnv->pQueryHandle = NULL; pRuntimeEnv->pTsdbReadHandle = NULL;
SMemRef* pMemRef = &pQueryAttr->memRef; SMemRef* pMemRef = &pQueryAttr->memRef;
assert(pMemRef->ref == 0 && pMemRef->snapshot.imem == NULL && pMemRef->snapshot.mem == NULL); assert(pMemRef->ref == 0 && pMemRef->snapshot.imem == NULL && pMemRef->snapshot.mem == NULL);
...@@ -3148,10 +3148,10 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa ...@@ -3148,10 +3148,10 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa
} else if ((*status) == BLK_DATA_STATIS_NEEDED) { } else if ((*status) == BLK_DATA_STATIS_NEEDED) {
// this function never returns error? // this function never returns error?
pCost->loadBlockStatis += 1; pCost->loadBlockStatis += 1;
tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockStatis); tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockStatis);
if (pBlock->pBlockStatis == NULL) { // data block statistics does not exist, load data block if (pBlock->pBlockStatis == NULL) { // data block statistics does not exist, load data block
pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL);
pCost->totalCheckedRows += pBlock->info.rows; pCost->totalCheckedRows += pBlock->info.rows;
} }
} else { } else {
...@@ -3159,7 +3159,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa ...@@ -3159,7 +3159,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa
// load the data block statistics to perform further filter // load the data block statistics to perform further filter
pCost->loadBlockStatis += 1; pCost->loadBlockStatis += 1;
tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockStatis); tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockStatis);
if (pQueryAttr->topBotQuery && pBlock->pBlockStatis != NULL) { if (pQueryAttr->topBotQuery && pBlock->pBlockStatis != NULL) {
{ // set previous window { // set previous window
...@@ -3205,7 +3205,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa ...@@ -3205,7 +3205,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa
pCost->totalCheckedRows += pBlockInfo->rows; pCost->totalCheckedRows += pBlockInfo->rows;
pCost->loadBlocks += 1; pCost->loadBlocks += 1;
pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL);
if (pBlock->pDataBlock == NULL) { if (pBlock->pDataBlock == NULL) {
return terrno; return terrno;
} }
...@@ -4494,7 +4494,7 @@ void queryCostStatis(SQInfo *pQInfo) { ...@@ -4494,7 +4494,7 @@ void queryCostStatis(SQInfo *pQInfo) {
// //
// assert(pQueryAttr->pos >= 0 && pQueryAttr->pos <= pBlockInfo->rows - 1); // assert(pQueryAttr->pos >= 0 && pQueryAttr->pos <= pBlockInfo->rows - 1);
// //
// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); // SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL);
// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0);
// //
// // update the pQueryAttr->limit.offset value, and pQueryAttr->pos value // // update the pQueryAttr->limit.offset value, and pQueryAttr->pos value
...@@ -4521,15 +4521,15 @@ void queryCostStatis(SQInfo *pQInfo) { ...@@ -4521,15 +4521,15 @@ void queryCostStatis(SQInfo *pQInfo) {
// int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); // int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order);
// //
// STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; // STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current;
// TsdbQueryHandleT pQueryHandle = pRuntimeEnv->pQueryHandle; // TsdbQueryHandleT pTsdbReadHandle = pRuntimeEnv->pTsdbReadHandle;
// //
// SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; // SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER;
// while (tsdbNextDataBlock(pQueryHandle)) { // while (tsdbNextDataBlock(pTsdbReadHandle)) {
// if (isQueryKilled(pRuntimeEnv->qinfo)) { // if (isQueryKilled(pRuntimeEnv->qinfo)) {
// longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); // longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED);
// } // }
// //
// tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); // tsdbRetrieveDataBlockInfo(pTsdbReadHandle, &blockInfo);
// //
// if (pQueryAttr->limit.offset > blockInfo.rows) { // if (pQueryAttr->limit.offset > blockInfo.rows) {
// pQueryAttr->limit.offset -= blockInfo.rows; // pQueryAttr->limit.offset -= blockInfo.rows;
...@@ -4562,7 +4562,7 @@ void queryCostStatis(SQInfo *pQInfo) { ...@@ -4562,7 +4562,7 @@ void queryCostStatis(SQInfo *pQInfo) {
// //
// // load the data block and check data remaining in current data block // // load the data block and check data remaining in current data block
// // TODO optimize performance // // TODO optimize performance
// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); // SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL);
// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0);
// //
// tw = *win; // tw = *win;
...@@ -4627,8 +4627,8 @@ void queryCostStatis(SQInfo *pQInfo) { ...@@ -4627,8 +4627,8 @@ void queryCostStatis(SQInfo *pQInfo) {
// STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; // STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current;
// //
// SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; // SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER;
// while (tsdbNextDataBlock(pRuntimeEnv->pQueryHandle)) { // while (tsdbNextDataBlock(pRuntimeEnv->pTsdbReadHandle)) {
// tsdbRetrieveDataBlockInfo(pRuntimeEnv->pQueryHandle, &blockInfo); // tsdbRetrieveDataBlockInfo(pRuntimeEnv->pTsdbReadHandle, &blockInfo);
// //
// if (QUERY_IS_ASC_QUERY(pQueryAttr)) { // if (QUERY_IS_ASC_QUERY(pQueryAttr)) {
// if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { // if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) {
...@@ -4674,7 +4674,7 @@ void queryCostStatis(SQInfo *pQInfo) { ...@@ -4674,7 +4674,7 @@ void queryCostStatis(SQInfo *pQInfo) {
// */ // */
// if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) { // if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) {
// //
// SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); // SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL);
// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0);
// //
// if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) { // if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) {
...@@ -4748,7 +4748,7 @@ static int32_t setupQueryHandle(void* tsdb, SQueryRuntimeEnv* pRuntimeEnv, int64 ...@@ -4748,7 +4748,7 @@ static int32_t setupQueryHandle(void* tsdb, SQueryRuntimeEnv* pRuntimeEnv, int64
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
if (isFirstLastRowQuery(pQueryAttr)) { if (isFirstLastRowQuery(pQueryAttr)) {
pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); pRuntimeEnv->pTsdbReadHandle = tsdbQueryLastRow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef);
// update the query time window // update the query time window
pQueryAttr->window = cond.twindow; pQueryAttr->window = cond.twindow;
...@@ -4769,11 +4769,11 @@ static int32_t setupQueryHandle(void* tsdb, SQueryRuntimeEnv* pRuntimeEnv, int64 ...@@ -4769,11 +4769,11 @@ static int32_t setupQueryHandle(void* tsdb, SQueryRuntimeEnv* pRuntimeEnv, int64
} }
} }
} else if (isCachedLastQuery(pQueryAttr)) { } else if (isCachedLastQuery(pQueryAttr)) {
pRuntimeEnv->pQueryHandle = tsdbQueryCacheLast(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); pRuntimeEnv->pTsdbReadHandle = tsdbQueryCacheLast(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef);
} else if (pQueryAttr->pointInterpQuery) { } else if (pQueryAttr->pointInterpQuery) {
pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); pRuntimeEnv->pTsdbReadHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef);
} else { } else {
pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); pRuntimeEnv->pTsdbReadHandle = tsdbQueryTables(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef);
} }
return terrno; return terrno;
...@@ -4831,19 +4831,19 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr ...@@ -4831,19 +4831,19 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr
switch(tbScanner) { switch(tbScanner) {
case OP_TableBlockInfoScan: { case OP_TableBlockInfoScan: {
pRuntimeEnv->proot = createTableBlockInfoScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); pRuntimeEnv->proot = createTableBlockInfoScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv);
break; break;
} }
case OP_TableSeqScan: { case OP_TableSeqScan: {
pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv);
break; break;
} }
case OP_DataBlocksOptScan: { case OP_DataBlocksOptScan: {
pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0); pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0);
break; break;
} }
case OP_TableScan: { case OP_TableScan: {
pRuntimeEnv->proot = createTableScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr)); pRuntimeEnv->proot = createTableScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr));
break; break;
} }
default: { // do nothing default: { // do nothing
...@@ -4974,13 +4974,13 @@ static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { ...@@ -4974,13 +4974,13 @@ static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) {
*newgroup = false; *newgroup = false;
while (tsdbNextDataBlock(pTableScanInfo->pQueryHandle)) { while (tsdbNextDataBlock(pTableScanInfo->pTsdbReadHandle)) {
if (isQueryKilled(pOperator->pRuntimeEnv->qinfo)) { if (isQueryKilled(pOperator->pRuntimeEnv->qinfo)) {
longjmp(pOperator->pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); longjmp(pOperator->pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED);
} }
pTableScanInfo->numOfBlocks += 1; pTableScanInfo->numOfBlocks += 1;
tsdbRetrieveDataBlockInfo(pTableScanInfo->pQueryHandle, &pBlock->info); tsdbRetrieveDataBlockInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->info);
// todo opt // todo opt
if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) { if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) {
...@@ -5037,7 +5037,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { ...@@ -5037,7 +5037,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) {
} }
if (++pTableScanInfo->current >= pTableScanInfo->times) { if (++pTableScanInfo->current >= pTableScanInfo->times) {
if (pTableScanInfo->reverseTimes <= 0 || isTsdbCacheLastRow(pTableScanInfo->pQueryHandle)) { if (pTableScanInfo->reverseTimes <= 0 || isTsdbCacheLastRow(pTableScanInfo->pTsdbReadHandle)) {
return NULL; return NULL;
} else { } else {
break; break;
...@@ -5046,7 +5046,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { ...@@ -5046,7 +5046,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) {
// do prepare for the next round table scan operation // do prepare for the next round table scan operation
STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window);
tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond);
setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED);
pRuntimeEnv->scanFlag = REPEAT_SCAN; pRuntimeEnv->scanFlag = REPEAT_SCAN;
...@@ -5069,7 +5069,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { ...@@ -5069,7 +5069,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) {
setupEnvForReverseScan(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); setupEnvForReverseScan(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput);
STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window);
tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond);
qDebug("QInfo:0x%"PRIx64" start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, qDebug("QInfo:0x%"PRIx64" start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64,
GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey); GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey);
...@@ -5112,8 +5112,8 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { ...@@ -5112,8 +5112,8 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) {
tableBlockDist.maxRows = INT_MIN; tableBlockDist.maxRows = INT_MIN;
tableBlockDist.minRows = INT_MAX; tableBlockDist.minRows = INT_MAX;
tsdbGetFileBlocksDistInfo(pTableScanInfo->pQueryHandle, &tableBlockDist); tsdbGetFileBlocksDistInfo(pTableScanInfo->pTsdbReadHandle, &tableBlockDist);
tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pQueryHandle); tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pTsdbReadHandle);
SSDataBlock* pBlock = &pTableScanInfo->block; SSDataBlock* pBlock = &pTableScanInfo->block;
pBlock->info.rows = 1; pBlock->info.rows = 1;
...@@ -5142,7 +5142,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* ...@@ -5142,7 +5142,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv*
assert(repeatTime > 0); assert(repeatTime > 0);
STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo));
pInfo->pQueryHandle = pTsdbQueryHandle; pInfo->pTsdbReadHandle = pTsdbQueryHandle;
pInfo->times = repeatTime; pInfo->times = repeatTime;
pInfo->reverseTimes = 0; pInfo->reverseTimes = 0;
pInfo->order = pRuntimeEnv->pQueryAttr->order.order; pInfo->order = pRuntimeEnv->pQueryAttr->order.order;
...@@ -5165,7 +5165,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* ...@@ -5165,7 +5165,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv*
SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) { SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) {
STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo));
pInfo->pQueryHandle = pTsdbQueryHandle; pInfo->pTsdbReadHandle = pTsdbQueryHandle;
pInfo->times = 1; pInfo->times = 1;
pInfo->reverseTimes = 0; pInfo->reverseTimes = 0;
pInfo->order = pRuntimeEnv->pQueryAttr->order.order; pInfo->order = pRuntimeEnv->pQueryAttr->order.order;
...@@ -5189,7 +5189,7 @@ SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeE ...@@ -5189,7 +5189,7 @@ SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeE
SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) { SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) {
STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo));
pInfo->pQueryHandle = pTsdbQueryHandle; pInfo->pTsdbReadHandle = pTsdbQueryHandle;
pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
SColumnInfoData infoData = {{0}}; SColumnInfoData infoData = {{0}};
...@@ -5271,7 +5271,7 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntime ...@@ -5271,7 +5271,7 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntime
assert(repeatTime > 0); assert(repeatTime > 0);
STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo));
pInfo->pQueryHandle = pTsdbQueryHandle; pInfo->pTsdbReadHandle = pTsdbQueryHandle;
pInfo->times = repeatTime; pInfo->times = repeatTime;
pInfo->reverseTimes = reverseTime; pInfo->reverseTimes = reverseTime;
pInfo->current = 0; pInfo->current = 0;
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或